00000800:00000200:1.1F:1775773497.947979:0:301880:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:1.1:1775773497.948010:0:301880:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:1.1:1775773497.948013:0:301880:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000800:00000200:1.1:1775773499.740359:0:301889:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1F:1775773499.740437:0:301890:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773501.415738:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773501.415740:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 1, 1 secs later) 00000800:00000200:1.1:1775773501.422356:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773501.422358:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 1, 1 secs later) 00000800:00000200:1.1:1775773504.478234:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773504.478625:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773504.478935:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773504.479084:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773504.479244:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773509.980447:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773509.980858:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773509.981060:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773509.981198:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773509.981295:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773515.100488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773515.100841:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773515.101120:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773515.101254:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773515.101355:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773520.220431:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773520.220437:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 8, 5 secs later) 00000800:00000200:1.1:1775773520.221144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773520.221146:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 8, 5 secs later) 00000800:00000200:1.1:1775773520.221361:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773520.221366:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 8, 5 secs later) 00000800:00000200:1.1:1775773520.221601:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773520.221602:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 8, 5 secs later) 00000800:00000200:1.1:1775773520.221953:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773520.221955:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 8, 5 secs later) 00000800:00000200:1.1:1775773525.340568:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773525.340975:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773525.341243:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773525.341412:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773525.341608:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773530.460671:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773530.460677:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 12 secs later) 00000800:00000200:1.1:1775773530.461661:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773530.461664:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 12 secs later) 00000800:00000200:1.1:1775773530.461932:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773530.461935:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 12 secs later) 00000800:00000200:1.1:1775773530.462200:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773530.462203:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 12 secs later) 00000800:00000200:1.1:1775773530.462689:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773530.462692:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 12 secs later) 00000800:00000200:3.1F:1775773535.580705:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773535.580715:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 6 secs later) 00000800:00000200:3.1:1775773535.581632:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773535.581634:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 6 secs later) 00000800:00000200:3.1:1775773535.581894:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773535.581896:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 6 secs later) 00000800:00000200:3.1:1775773535.582139:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773535.582141:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 6 secs later) 00000800:00000200:3.1:1775773535.582405:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773535.582407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 6 secs later) 00000800:00000200:3.1:1775773540.700385:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773540.700401:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 1 secs later) 00000800:00000200:3.1:1775773540.701011:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773540.701012:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 1 secs later) 00000800:00000200:3.1:1775773540.701205:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773540.701206:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 1 secs later) 00000800:00000200:3.1:1775773540.701412:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773540.701414:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 1 secs later) 00000800:00000200:3.1:1775773540.701631:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773540.701632:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 16, 1 secs later) 00000800:00000200:3.1:1775773545.820394:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773545.820691:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773545.820947:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773545.821055:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773545.821146:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773550.940754:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773550.940771:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 28 secs later) 00000800:00000200:3.1:1775773550.941607:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773550.941609:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 28 secs later) 00000800:00000200:3.1:1775773550.941905:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773550.941907:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 28 secs later) 00000800:00000200:3.1:1775773550.942114:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773550.942115:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 28 secs later) 00000800:00000200:3.1:1775773550.942326:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773550.942329:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 28 secs later) 00000800:00000200:3.1:1775773556.060577:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773556.060585:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 23 secs later) 00000800:00000200:3.1:1775773556.061219:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773556.061221:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 23 secs later) 00000800:00000200:3.1:1775773556.061541:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773556.061544:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 23 secs later) 00000800:00000200:3.1:1775773556.061883:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773556.061886:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 23 secs later) 00000800:00000200:3.1:1775773556.062215:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773556.062218:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 23 secs later) 00000800:00000200:3.1:1775773561.180324:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773561.180328:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 18 secs later) 00000800:00000200:3.1:1775773561.180722:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773561.180723:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 18 secs later) 00000800:00000200:3.1:1775773561.180954:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773561.180959:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 18 secs later) 00000800:00000200:3.1:1775773561.181164:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773561.181166:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 18 secs later) 00000800:00000200:3.1:1775773561.181427:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773561.181429:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 18 secs later) 00000800:00000200:3.1:1775773566.300449:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773566.300455:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 13 secs later) 00000800:00000200:3.1:1775773566.301194:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773566.301196:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 13 secs later) 00000800:00000200:3.1:1775773566.301425:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773566.301427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 13 secs later) 00000800:00000200:3.1:1775773566.301706:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773566.301709:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 13 secs later) 00000800:00000200:3.1:1775773566.302129:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773566.302131:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 13 secs later) 00000800:00000200:3.1:1775773571.420474:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773571.420479:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 8 secs later) 00000800:00000200:3.1:1775773571.421188:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773571.421190:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 8 secs later) 00000800:00000200:3.1:1775773571.421439:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773571.421440:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 8 secs later) 00000800:00000200:3.1:1775773571.421718:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773571.421721:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 8 secs later) 00000800:00000200:3.1:1775773571.422126:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773571.422134:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 8 secs later) 00000800:00000200:3.1:1775773576.540765:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773576.540774:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 3 secs later) 00000800:00000200:3.1:1775773576.541650:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773576.541654:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 3 secs later) 00000800:00000200:3.1:1775773576.542019:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773576.542021:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 3 secs later) 00000800:00000200:3.1:1775773576.542358:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773576.542360:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 3 secs later) 00000800:00000200:3.1:1775773576.542791:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773576.542794:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 32, 3 secs later) 00000800:00000200:3.1:1775773581.660451:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773581.660874:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773581.661021:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773581.661130:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773581.661228:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773586.780402:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773586.780406:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:3.1:1775773586.781071:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773586.781073:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:3.1:1775773586.781329:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773586.781331:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:3.1:1775773586.781587:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773586.781589:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:3.1:1775773586.781964:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773586.781968:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:3.1:1775773591.900347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773591.900352:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773591.901072:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773591.901074:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773591.901314:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773591.901316:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773591.901611:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773591.901613:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773591.901895:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773591.901897:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773597.020814:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773597.020818:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773597.021519:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773597.021523:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773597.021896:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773597.021899:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773597.022284:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773597.022287:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773597.022657:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773597.022661:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773602.140894:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773602.140902:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773602.141968:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773602.141975:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773602.142298:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773602.142301:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773602.142651:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773602.142654:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773602.143134:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773602.143136:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773607.260588:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773607.260594:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773607.261433:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773607.261436:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773607.261765:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773607.261771:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773607.262082:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773607.262084:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773607.262279:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773607.262281:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773612.380466:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773612.380472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773612.381191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773612.381194:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773612.381454:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773612.381456:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773612.381706:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773612.381708:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773612.382008:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773612.382011:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773617.500514:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773617.500520:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773617.501003:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773617.501005:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773617.501297:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773617.501299:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773617.501518:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773617.501520:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773617.501757:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773617.501763:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773622.620585:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773622.620591:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 21 secs later) 00000800:00000200:1.1:1775773622.621204:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773622.621206:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 21 secs later) 00000800:00000200:1.1:1775773622.621440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773622.621442:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 21 secs later) 00000800:00000200:1.1:1775773622.621648:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773622.621650:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 21 secs later) 00000800:00000200:1.1:1775773622.621870:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773622.621872:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 21 secs later) 00000800:00000200:1.1:1775773627.741098:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773627.741105:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1775773627.742124:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773627.742126:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1775773627.742492:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773627.742495:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1775773627.742816:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773627.742819:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1775773627.743225:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773627.743227:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1775773632.860543:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773632.860548:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773632.861227:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773632.861231:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773632.861436:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773632.861437:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773632.861629:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773632.861630:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773632.861870:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773632.861872:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773637.981434:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773637.981457:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773637.982710:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773637.982715:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773637.983343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773637.983346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773637.983802:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773637.983807:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773637.984342:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773637.984344:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773643.100629:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773643.100650:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773643.101279:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773643.101282:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773643.101688:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773643.101691:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773643.102164:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773643.102189:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773643.102554:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773643.102557:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773648.220650:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773648.221174:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773648.221412:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773648.221601:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773648.221773:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773653.340504:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773653.340508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773653.341305:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773653.341307:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773653.341654:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773653.341656:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773653.341989:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773653.341990:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773653.342271:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773653.342274:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773658.460538:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773658.460544:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773658.461320:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773658.461322:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773658.461544:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773658.461546:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773658.461743:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773658.461748:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773658.462090:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773658.462093:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773663.580464:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773663.580471:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773663.581031:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773663.581033:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773663.581288:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773663.581290:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773663.581507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773663.581509:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773663.581710:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773663.581712:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773668.700511:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773668.700516:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773668.701321:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773668.701322:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773668.701525:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773668.701527:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773668.701885:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773668.701886:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773668.702247:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773668.702249:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773673.820312:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773673.820318:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773673.821000:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773673.821004:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773673.821247:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773673.821249:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773673.821490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773673.821492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773673.821706:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773673.821707:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773678.940373:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773678.940376:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773678.940978:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773678.940980:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773678.941200:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773678.941202:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773678.941476:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773678.941478:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773678.941761:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773678.941763:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773684.060629:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773684.060638:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773684.061886:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773684.061889:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773684.062270:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773684.062276:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773684.062640:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773684.062643:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773684.063070:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773684.063072:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773689.180649:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773689.180659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773689.181463:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773689.181465:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773689.181727:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773689.181730:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773689.182003:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773689.182005:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773689.182209:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773689.182211:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773694.300415:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773694.300419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773694.300903:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773694.300905:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773694.301107:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773694.301109:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773694.301304:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773694.301306:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773694.301500:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773694.301504:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773699.421504:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773699.421519:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773699.423048:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773699.423050:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773699.423408:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773699.423427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773699.423915:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773699.423918:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773699.424231:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773699.424233:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773704.540448:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773704.540455:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773704.541116:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773704.541118:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773704.541347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773704.541349:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773704.541548:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773704.541550:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773704.541863:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773704.541865:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773709.660888:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773709.660895:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773709.661537:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773709.661549:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773709.662009:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773709.662011:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773709.662368:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773709.662370:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773709.662721:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773709.662723:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773714.780521:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773714.780997:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773714.781297:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773714.781521:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773714.781705:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773719.900511:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773719.900518:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773719.901294:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773719.901297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773719.901629:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773719.901632:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773719.902011:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773719.902013:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773719.902360:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773719.902362:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773725.020494:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773725.020499:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773725.021046:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773725.021051:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773725.021270:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773725.021272:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773725.021462:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773725.021464:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773725.021673:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773725.021676:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773730.140704:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773730.140709:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773730.141325:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773730.141326:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773730.141671:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773730.141673:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773730.141924:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773730.141927:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773730.142126:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773730.142127:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773735.260479:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773735.260484:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773735.261270:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773735.261273:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773735.261752:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773735.261754:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773735.262196:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773735.262198:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773735.262427:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773735.262429:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773740.380440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773740.380444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773740.381084:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773740.381086:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773740.381288:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773740.381290:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773740.381477:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773740.381482:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773740.381739:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773740.381741:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773745.500495:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773745.500500:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775773745.501273:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773745.501276:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775773745.501566:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773745.501569:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775773745.501928:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773745.501930:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775773745.502273:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773745.502276:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775773750.620754:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773750.620758:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773750.621946:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773750.621949:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773750.622469:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773750.622472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773750.622825:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773750.622831:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773750.623048:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773750.623050:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773755.740353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773755.740357:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773755.741074:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773755.741076:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773755.741275:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773755.741277:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773755.741471:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773755.741472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773755.741773:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773755.741775:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773760.860721:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773760.860726:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773760.861489:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773760.861491:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773760.861722:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773760.861723:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773760.861947:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773760.861949:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773760.862146:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773760.862167:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773765.980597:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773765.980604:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773765.981362:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773765.981364:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773765.981690:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773765.981691:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773765.982004:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773765.982006:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773765.982282:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773765.982284:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773771.100838:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773771.100863:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773771.101383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773771.101384:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773771.101608:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773771.101610:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773771.101860:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773771.101863:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773771.102191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773771.102193:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773776.220938:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773776.220944:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773776.221969:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773776.221972:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773776.222376:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773776.222379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773776.222748:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773776.222751:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773776.223156:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773776.223160:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773781.340550:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773781.340994:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773781.341361:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773781.341554:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773781.341734:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773786.460761:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773786.460777:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773786.461525:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773786.461527:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773786.461965:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773786.461966:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773786.462295:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773786.462297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773786.462524:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773786.462526:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773791.580627:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773791.580638:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773791.581299:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773791.581301:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773791.581578:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773791.581581:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773791.581805:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773791.581807:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773791.582234:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773791.582237:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773796.700426:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773796.700432:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773796.701114:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773796.701116:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773796.701320:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773796.701322:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773796.701543:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773796.701544:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773796.701867:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773796.701870:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773801.820392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773801.820397:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773801.821009:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773801.821010:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773801.821209:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773801.821214:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773801.821471:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773801.821472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773801.821766:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773801.821767:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773806.940380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773806.940386:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773806.941117:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773806.941120:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773806.941377:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773806.941379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773806.941642:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773806.941645:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773806.941961:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773806.941963:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773812.060418:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773812.060423:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773812.060981:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773812.060982:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773812.061206:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773812.061208:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773812.061402:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773812.061404:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773812.061591:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773812.061595:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773817.180542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773817.180547:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773817.181151:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773817.181153:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773817.181460:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773817.181462:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773817.181787:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773817.181790:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773817.182077:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773817.182079:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773822.300418:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773822.300422:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773822.301100:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773822.301101:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773822.301299:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773822.301301:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773822.301579:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773822.301581:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773822.301916:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773822.301919:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773827.421317:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773827.421340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773827.422207:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773827.422210:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773827.422993:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773827.422996:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773827.423510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773827.423512:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773827.424043:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773827.424046:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773832.540464:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773832.540469:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773832.541224:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773832.541226:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773832.541520:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773832.541525:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773832.541738:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773832.541741:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773832.542060:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773832.542061:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773837.660462:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773837.660466:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773837.661073:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773837.661076:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773837.661572:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773837.661573:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773837.662003:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773837.662005:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773837.662297:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773837.662299:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773842.780588:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773842.780592:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773842.781242:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773842.781244:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773842.781491:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773842.781493:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773842.781717:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773842.781719:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773842.782013:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773842.782021:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773847.900434:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773847.900777:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773847.901167:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773847.901436:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773847.901603:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773853.020591:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773853.020596:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773853.021407:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773853.021410:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773853.021812:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773853.021815:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773853.022183:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773853.022186:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773853.022484:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773853.022486:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773858.140476:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773858.140479:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773858.141189:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773858.141190:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773858.141471:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773858.141473:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773858.141711:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773858.141713:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773858.142038:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773858.142042:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773863.260475:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773863.260480:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773863.261165:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773863.261168:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773863.261373:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773863.261376:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773863.261649:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773863.261651:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773863.261982:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773863.261984:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775773868.380404:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773868.380409:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775773868.380956:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773868.380958:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775773868.381190:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773868.381192:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775773868.381412:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773868.381414:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775773868.381586:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773868.381588:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775773873.500434:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773873.500439:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775773873.501005:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773873.501013:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775773873.501201:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773873.501203:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775773873.501374:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773873.501376:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775773873.501543:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773873.501545:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775773878.620475:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773878.620481:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773878.621110:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773878.621111:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773878.621323:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773878.621325:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773878.621505:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773878.621507:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773878.621811:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773878.621813:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773883.740362:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773883.740366:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773883.740982:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773883.740987:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773883.741183:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773883.741185:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773883.741406:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773883.741408:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773883.741620:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773883.741622:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773888.860422:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773888.860427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773888.861052:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773888.861055:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773888.861349:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773888.861351:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773888.861662:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773888.861665:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773888.861943:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773888.861945:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773893.980394:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773893.980401:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773893.981030:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773893.981032:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773893.981267:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773893.981268:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773893.981520:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773893.981527:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773893.981814:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773893.981815:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773899.100575:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773899.100582:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773899.101131:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773899.101132:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773899.101429:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773899.101432:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773899.101715:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773899.101717:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773899.101985:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773899.101987:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773904.220646:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773904.220651:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775773904.221724:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773904.221725:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775773904.221956:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773904.221958:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775773904.222189:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773904.222191:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775773904.222499:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773904.222501:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773909.340837:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773909.340843:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773909.341768:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773909.341773:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773909.342111:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773909.342113:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773909.342413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773909.342424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773909.342706:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773909.342708:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773914.460424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773914.460812:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773914.461074:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773914.461227:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773914.461370:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773919.580420:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773919.580424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1775773919.581876:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773919.581881:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1775773919.582425:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773919.582427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1775773919.582653:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773919.582655:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1775773919.582962:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773919.582965:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1775773924.700500:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773924.700506:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775773924.701186:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773924.701187:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775773924.701389:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773924.701391:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775773924.701570:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773924.701577:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775773924.701824:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773924.701826:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773929.820485:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773929.820492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773929.821088:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773929.821091:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773929.821362:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773929.821363:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773929.821606:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773929.821607:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773929.821859:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773929.821861:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773934.940392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773934.940398:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773934.940975:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773934.940979:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773934.941444:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773934.941446:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773934.941694:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773934.941695:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773934.941897:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773934.941898:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773940.060686:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773940.060696:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773940.061566:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773940.061568:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773940.061825:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773940.061828:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773940.062043:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773940.062045:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773940.062238:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773940.062240:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773945.180697:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773945.180702:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773945.181504:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773945.181506:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773945.181706:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773945.181712:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773945.181971:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773945.181973:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773945.182179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773945.182181:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773950.300583:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773950.300593:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773950.301570:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773950.301572:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773950.301801:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773950.301802:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773950.302040:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773950.302041:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773950.302263:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773950.302265:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773955.420488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773955.420494:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773955.421079:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773955.421081:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773955.421309:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773955.421310:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773955.421555:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773955.421557:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773955.421836:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773955.421843:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773960.540599:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773960.540605:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773960.541258:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773960.541260:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773960.541482:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773960.541484:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773960.541669:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773960.541670:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773960.541914:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773960.541916:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773965.660411:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773965.660425:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773965.661117:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773965.661120:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773965.661444:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773965.661447:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773965.661701:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773965.661703:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773965.661986:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773965.661987:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775773970.780542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773970.780548:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773970.781278:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773970.781296:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773970.781590:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773970.781593:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773970.781893:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773970.781897:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773970.782337:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773970.782339:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775773975.900581:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773975.900588:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773975.901501:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773975.901508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773975.901745:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773975.901759:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773975.901992:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773975.901993:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775773975.902240:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775773975.902242:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775773981.020483:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773981.020965:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773981.021134:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773981.021313:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773981.021502:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773986.140489:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773986.140495:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773986.141296:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773986.141300:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773986.141596:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773986.141598:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773986.141909:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773986.141911:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773986.142204:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773986.142206:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773991.260495:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773991.260501:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775773991.261177:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773991.261179:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775773991.261401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773991.261402:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775773991.261610:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773991.261612:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775773991.261882:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773991.261888:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775773996.380413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773996.380418:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773996.381163:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773996.381165:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773996.381412:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773996.381414:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773996.381614:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773996.381615:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773996.381889:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775773996.381891:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774001.500488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774001.500494:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774001.501165:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774001.501169:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774001.501468:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774001.501470:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774001.501697:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774001.501699:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774001.501889:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774001.501891:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774006.620448:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774006.620452:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774006.621025:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774006.621027:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774006.621269:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774006.621272:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774006.621586:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774006.621588:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774006.621931:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774006.621947:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775774011.740514:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774011.740518:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774011.741053:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774011.741055:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774011.741257:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774011.741260:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774011.741456:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774011.741468:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774011.741657:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774011.741659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774016.860640:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774016.860649:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774016.861673:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774016.861688:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774016.862041:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774016.862044:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774016.862347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774016.862349:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774016.862812:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774016.862815:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774021.980523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774021.980529:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774021.981113:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774021.981114:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774021.981340:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774021.981342:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774021.981581:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774021.981582:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774021.981834:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774021.981836:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774027.100432:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774027.100438:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774027.101074:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774027.101076:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774027.101259:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774027.101260:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774027.101431:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774027.101433:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774027.101603:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774027.101604:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774032.220611:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774032.220616:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774032.221407:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774032.221408:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774032.221597:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774032.221599:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774032.221895:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774032.221898:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774032.222290:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774032.222293:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774037.340495:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774037.340501:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774037.341222:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774037.341225:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774037.341549:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774037.341552:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774037.341909:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774037.341929:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774037.342373:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774037.342375:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00010000:00000010:0.1:1775774040.164814:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d53b63b00. 00010000:00000010:1.1:1775774040.164814:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d5123c200. 00010000:00000010:0.1:1775774040.164822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d52855e00. 00010000:00000010:1.1:1775774040.164822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcd000. 00010000:00000010:1.1:1775774040.164826:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d5123c100. 00010000:00000010:0.1:1775774040.164827:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d66c49700. 00010000:00000010:0.1:1775774040.164830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d52854400. 00010000:00000010:1.1:1775774040.164830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcda00. 00010000:00000010:0.1:1775774040.164833:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d66c49f00. 00010000:00000010:0.1:1775774040.164835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d52854a00. 00010000:00000010:0.1:1775774040.164838:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d5123c000. 00010000:00000010:0.1:1775774040.164842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcde00. 00010000:00000010:1.1:1775774040.164842:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d5123c700. 00010000:00000010:0.1:1775774040.164845:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d5123c300. 00010000:00000010:1.1:1775774040.164845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcce00. 00010000:00000010:0.1:1775774040.164847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcd600. 00010000:00000010:1.1:1775774040.164848:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d53b63a00. 00010000:00000010:0.1:1775774040.164850:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d58a64200. 00010000:00000010:1.1:1775774040.164851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d52855600. 00010000:00000010:0.1:1775774040.164853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcd800. 00010000:00000010:1.1:1775774040.164854:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d5123ca00. 00010000:00000010:0.1:1775774040.164856:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d45288200. 00010000:00000010:1.1:1775774040.164857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcca00. 00010000:00000010:0.1:1775774040.164859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dccc00. 00010000:00000010:1.1:1775774040.164859:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d45288f00. 00010000:00000010:1.1:1775774040.164862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcd400. 00010000:00000010:1.1:1775774040.164864:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff978d45288d00. 00010000:00000010:1.1:1775774040.164867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff978d45dcc200. 00000800:00000200:1.1:1775774042.460324:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774042.460327:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774042.460763:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774042.460765:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774042.461033:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774042.461035:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774042.461244:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774042.461246:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774042.461566:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774042.461568:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774047.580457:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774047.580869:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774047.581027:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774047.581204:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774047.581365:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774052.700452:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774052.700459:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774052.701260:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774052.701279:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774052.701621:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774052.701624:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774052.701951:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774052.701953:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774052.702261:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774052.702264:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775774057.820496:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774057.820502:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774057.821241:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774057.821242:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774057.821510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774057.821512:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774057.821765:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774057.821767:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774057.822063:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774057.822065:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774062.940375:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774062.940379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774062.940988:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774062.940991:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774062.941425:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774062.941427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774062.941921:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774062.941924:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774062.942376:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774062.942379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774068.060353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774068.060358:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774068.061016:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774068.061019:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774068.061310:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774068.061311:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774068.061581:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774068.061589:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774068.061968:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774068.061970:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774073.180455:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774073.180460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774073.181187:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774073.181190:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774073.181515:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774073.181516:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774073.181774:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774073.181776:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774073.182021:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774073.182023:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774078.300577:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774078.300582:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774078.301430:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774078.301433:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774078.301672:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774078.301674:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774078.301932:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774078.301934:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774078.302173:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774078.302175:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774083.420467:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774083.420482:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774083.421233:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774083.421236:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774083.421488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774083.421490:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774083.421668:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774083.421670:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774083.421862:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774083.421865:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775774088.540386:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774088.540392:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775774088.541020:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774088.541022:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775774088.541234:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774088.541236:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775774088.541413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774088.541415:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775774088.541584:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774088.541585:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775774093.660489:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774093.660507:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774093.661077:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774093.661079:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774093.661295:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774093.661298:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774093.661614:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774093.661617:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774093.661874:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774093.661875:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774098.780469:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774098.780474:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774098.781132:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774098.781133:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774098.781366:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774098.781369:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774098.781673:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774098.781676:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774098.782114:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774098.782116:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774103.900369:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774103.900376:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774103.900978:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774103.900980:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774103.901159:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774103.901161:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774103.901391:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774103.901393:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774103.901677:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774103.901680:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774109.020503:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774109.020508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774109.021121:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774109.021123:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774109.021377:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774109.021387:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774109.021626:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774109.021627:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774109.021891:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774109.021892:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774114.140462:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774114.140755:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774114.141081:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774114.141289:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774114.141453:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774119.260799:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774119.260803:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774119.261133:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774119.261135:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774119.261415:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774119.261417:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774119.261681:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774119.261683:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774119.262521:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774119.262523:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774124.380401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774124.380406:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774124.381018:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774124.381020:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774124.381204:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774124.381210:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774124.381407:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774124.381409:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774124.381625:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774124.381626:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774129.500410:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774129.500416:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774129.500962:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774129.500964:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774129.501221:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774129.501223:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774129.501467:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774129.501469:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774129.501703:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774129.501705:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774134.620281:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774134.620287:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774134.620846:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774134.620848:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774134.621124:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774134.621126:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774134.621383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774134.621385:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774134.621647:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774134.621659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774139.740370:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774139.740374:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774139.740716:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774139.740717:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774139.740937:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774139.740939:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774139.741135:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774139.741137:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774139.741328:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774139.741330:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774144.860407:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774144.860411:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774144.860915:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774144.860916:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774144.861123:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774144.861125:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774144.861311:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774144.861312:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774144.861594:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774144.861597:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774149.980449:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774149.980455:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774149.981275:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774149.981285:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774149.981598:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774149.981600:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774149.982025:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774149.982027:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774149.982274:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774149.982276:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774155.100566:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774155.100571:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774155.101333:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774155.101336:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774155.101637:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774155.101639:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774155.101946:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774155.101948:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774155.102346:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774155.102349:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774160.220689:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774160.220702:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774160.221496:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774160.221498:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774160.221762:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774160.221764:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774160.221976:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774160.221994:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774160.222185:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774160.222187:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774165.340741:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774165.340748:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774165.341266:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774165.341268:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774165.341576:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774165.341579:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774165.341904:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774165.341906:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774165.342194:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774165.342197:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774170.460507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774170.460514:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774170.461055:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774170.461058:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774170.461375:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774170.461377:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774170.461670:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774170.461671:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774170.461898:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774170.461900:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774175.580522:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774175.580534:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774175.581199:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774175.581204:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774175.581530:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774175.581532:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774175.581970:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774175.581973:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774175.582220:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774175.582222:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774180.700593:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774180.701005:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774180.701339:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774180.701533:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774180.701670:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774185.820747:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774185.820751:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774185.821698:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774185.821699:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774185.822007:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774185.822009:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774185.822272:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774185.822273:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774185.822495:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774185.822497:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774190.940384:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774190.940402:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774190.941073:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774190.941075:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774190.941295:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774190.941298:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774190.941517:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774190.941519:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774190.941838:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774190.941840:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774196.060500:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774196.060506:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774196.061233:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774196.061237:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774196.061442:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774196.061444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774196.061664:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774196.061667:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774196.062076:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774196.062078:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774201.180484:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774201.180489:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774201.181144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774201.181147:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774201.181460:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774201.181467:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774201.181735:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774201.181736:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774201.181985:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774201.181986:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774206.300375:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774206.300380:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774206.301176:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774206.301179:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774206.301482:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774206.301484:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774206.301799:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774206.301802:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774206.302224:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774206.302226:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774211.420566:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774211.420573:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774211.421247:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774211.421250:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774211.421550:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774211.421553:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774211.421889:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774211.421891:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774211.422189:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774211.422197:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774216.540394:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774216.540401:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774216.541114:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774216.541120:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774216.541345:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774216.541347:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774216.541574:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774216.541575:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774216.541879:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774216.541881:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774221.660490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774221.660495:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774221.661177:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774221.661181:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774221.661454:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774221.661457:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774221.661712:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774221.661714:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774221.661984:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774221.661986:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774226.780452:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774226.780471:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774226.781061:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774226.781068:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774226.781330:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774226.781333:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774226.781613:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774226.781616:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774226.781838:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:1.1:1775774226.781841:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774231.900453:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774231.900456:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774231.901082:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774231.901085:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774231.901303:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774231.901315:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774231.901545:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774231.901547:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774231.901916:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774231.901917:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774237.020429:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774237.020435:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774237.021019:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774237.021021:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774237.021338:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774237.021340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774237.021627:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774237.021630:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774237.021860:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774237.021862:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774242.140516:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774242.140522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774242.141181:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774242.141183:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774242.141397:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774242.141399:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774242.141612:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774242.141615:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774242.141987:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774242.141995:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774247.260460:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774247.260839:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774247.261087:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774247.261291:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774247.261452:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774390.620403:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774390.620407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775774390.620880:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774390.620883:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775774390.621059:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774390.621061:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775774390.621228:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774390.621230:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775774390.621431:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774390.621433:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775774395.740433:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774395.740436:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775774395.741110:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774395.741113:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775774395.741500:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774395.741502:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775774395.741771:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774395.741773:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775774395.742004:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774395.742011:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775774400.860510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774400.860518:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775774400.861137:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774400.861140:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775774400.861464:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774400.861467:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775774400.861800:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774400.861803:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775774400.862109:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774400.862111:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775774405.980415:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774405.980422:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775774405.981092:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774405.981094:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775774405.981294:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774405.981297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775774405.981537:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774405.981538:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775774405.981815:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774405.981818:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775774411.100504:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774411.100510:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774411.101163:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774411.101176:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774411.101499:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774411.101502:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774411.101793:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774411.101797:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774411.102024:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774411.102026:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774416.220466:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774416.220469:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774416.221084:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774416.221086:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774416.221293:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774416.221294:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774416.221497:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774416.221499:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774416.221731:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774416.221733:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774421.340489:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774421.340496:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774421.341148:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774421.341151:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774421.341469:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774421.341472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774421.341799:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774421.341815:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774421.342138:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774421.342140:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774431.580473:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774431.580479:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775774431.581085:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774431.581087:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775774431.581306:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774431.581308:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775774431.581666:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774431.581668:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775774431.582039:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774431.582041:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775774462.300459:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774462.300465:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774462.301029:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774462.301031:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774462.301258:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774462.301261:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774462.301472:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774462.301473:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774462.301680:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774462.301682:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774467.420343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774467.420359:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774467.420738:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774467.420740:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774467.420992:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774467.420994:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774467.421179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774467.421180:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774467.421370:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774467.421372:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774472.540419:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774472.540423:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774472.541120:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774472.541121:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774472.541335:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774472.541337:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774472.541619:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774472.541622:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774472.541922:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774472.541923:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774477.660463:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774477.660468:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774477.661112:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774477.661115:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774477.661408:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774477.661421:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774477.661605:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774477.661607:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774477.661809:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774477.661812:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775774482.780542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774482.780548:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774482.781185:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774482.781186:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774482.781373:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774482.781375:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774482.781549:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774482.781551:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774482.781833:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774482.781836:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775774487.900394:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774487.900400:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774487.900987:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774487.900989:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774487.901170:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774487.901171:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774487.901341:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774487.901342:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774487.901508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774487.901522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775774503.260395:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774503.260400:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774503.261126:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774503.261128:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774503.261383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774503.261385:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774503.261565:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774503.261567:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774503.261872:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774503.261874:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774508.380666:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774508.380673:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774508.381204:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774508.381207:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774508.381503:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774508.381506:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774508.381840:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774508.381843:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774508.382145:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774508.382147:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774513.500489:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774513.500854:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774513.501026:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774513.501190:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774513.501445:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774518.621071:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774518.621075:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775774518.621580:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774518.621583:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775774518.622015:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774518.622019:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775774518.622260:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774518.622262:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775774518.622472:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774518.622474:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775774523.740536:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774523.740543:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774523.741092:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774523.741094:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774523.741302:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774523.741304:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774523.741490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774523.741492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774523.741721:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774523.741724:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774528.860643:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774528.860649:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774528.861422:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774528.861424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774528.861750:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774528.861754:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774528.862087:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774528.862090:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774528.862410:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774528.862413:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774533.980426:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774533.980430:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774533.980973:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774533.980975:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774533.981179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774533.981181:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774533.981387:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774533.981389:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774533.981562:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774533.981564:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774539.100491:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774539.100497:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774539.101116:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774539.101118:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774539.101427:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774539.101430:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774539.101733:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774539.101736:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774539.102055:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774539.102057:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775774544.220500:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774544.220505:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775774544.221064:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774544.221065:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775774544.221253:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774544.221256:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775774544.221485:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774544.221487:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775774544.221754:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774544.221756:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775774549.340552:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774549.340561:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775774549.341541:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774549.341546:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000100:00000001:1.0:1775774549.341887:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774549.341898:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099515264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774549.341920:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774549.341933:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16274) 00000800:00000200:3.1:1775774549.342108:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774549.342110:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775774549.342358:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774549.342360:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775774549.342547:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774549.342548:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1775774550.108025:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774550.108030:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774550.108033:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774550.619931:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310938358, at 4310938368 00000100:00000010:1.0:1775774550.619946:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000020:00000001:1.0:1775774551.131983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774551.131986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774551.131990:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774552.156008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774552.156012:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774552.156016:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774553.179964:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774553.179967:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774553.179971:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774554.203944:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774554.203947:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774554.203950:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774554.459880:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774554.459888:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774554.459901:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774554.459904:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774554.459907:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774554.459913:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16269 00000400:00000200:1.0:1775774554.459922:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774554.459927:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774554.459930:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774554.459932:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.459934:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.459936:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.459938:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774554.459941:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774554.459943:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774554.459945:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774554.459948:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774554.459949:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774554.459951:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774554.459952:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.459954:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.459955:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.459956:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774554.459958:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774554.459960:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774554.459962:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774554.459963:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774554.459966:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774554.459968:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.459969:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774554.459971:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774554.459979:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774554.459982:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774554.459985:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774554.459986:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.459989:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774554.459991:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774554.459994:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774554.460002:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775774554.460007:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774554.460014:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099515392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774554.460022:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774554.460026:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774554.460030:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099515392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774554.460049:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774554.460051:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774554.460054:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774554.460055:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774554.460056:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774554.460058:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774554.460060:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16269 00000400:00000200:1.0:1775774554.460062:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774554.460064:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774554.460065:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774554.460067:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460068:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.460070:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460071:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774554.460072:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774554.460073:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774554.460096:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774554.460098:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774554.460100:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774554.460101:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774554.460104:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460105:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.460107:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460109:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774554.460111:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774554.460113:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460117:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774554.460118:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774554.460123:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774554.460127:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460128:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774554.460132:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774554.460135:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460138:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774554.460156:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774554.460159:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774554.460161:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774554.460164:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460167:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774554.460169:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774554.460172:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774554.460177:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775774554.460182:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774554.460190:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099515520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774554.460201:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774554.460203:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774554.460213:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099515520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774554.460229:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774554.460232:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774554.460238:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774554.460240:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774554.460242:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774554.460243:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774554.460246:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16269 00000400:00000200:1.0:1775774554.460252:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774554.460256:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774554.460260:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774554.460265:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460267:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.460271:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460282:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774554.460284:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774554.460285:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774554.460288:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774554.460290:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774554.460291:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774554.460293:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774554.460296:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460299:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.460303:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460307:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774554.460309:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774554.460314:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460319:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774554.460323:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774554.460326:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774554.460330:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460333:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774554.460335:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774554.460339:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460341:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774554.460347:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775774554.460351:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774554.460353:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774554.460354:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460358:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774554.460361:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774554.460362:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774554.460377:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774554.460382:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774554.460388:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099515648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774554.460398:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774554.460402:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774554.460409:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099515648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774554.460428:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774554.460430:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774554.460438:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774554.460442:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774554.460445:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774554.460451:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774554.460457:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16269 00000400:00000200:1.0:1775774554.460465:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774554.460473:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774554.460478:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1775774554.460478:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774554.460483:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000100:00000040:1.0:1775774554.460485:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460488:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.460490:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460492:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774554.460494:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774554.460496:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774554.460498:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774554.460501:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774554.460504:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774554.460507:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774554.460513:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460515:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.460517:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460522:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774554.460523:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774554.460530:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460532:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774554.460534:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774554.460538:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774554.460549:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460553:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774554.460556:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774554.460561:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460563:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774554.460567:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774554.460571:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774554.460577:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774554.460579:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460582:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774554.460584:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774554.460586:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774554.460591:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774554.460597:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774554.460605:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099515776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774554.460614:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774554.460616:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774554.460620:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099515776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774554.460626:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774554.460632:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774554.460637:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774554.460640:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774554.460642:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774554.460646:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774554.460650:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16269 00000400:00000200:1.0:1775774554.460655:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774554.460660:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774554.460664:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774554.460669:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460673:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.460677:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460679:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774554.460680:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774554.460682:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774554.460685:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774554.460686:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774554.460690:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774554.460694:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774554.460700:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460702:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774554.460706:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460710:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774554.460714:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774554.460720:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774554.460725:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774554.460726:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774554.460730:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774554.460735:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460739:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774554.460764:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774554.460771:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460775:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774554.460807:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774554.460812:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774554.460816:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774554.460820:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774554.460824:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774554.460827:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774554.460831:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774554.460838:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774554.460843:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774554.460854:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099515904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774554.460868:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774554.460872:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774554.460879:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099515904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774554.460890:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774554.460903:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16279) 00000800:00000200:3.1:1775774554.461115:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774554.461116:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775774554.461371:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774554.461374:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775774554.461601:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774554.461602:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775774554.461881:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774554.461884:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000020:00000001:1.0:1775774555.227935:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774555.227938:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774555.227940:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774555.739897:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310943477, at 4310943488 00000100:00000010:1.0:1775774555.739911:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344360. 00000020:00000001:1.0:1775774556.251999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774556.252004:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774556.252008:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774557.275947:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774557.275952:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774557.275956:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774558.300034:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774558.300037:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774558.300041:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774559.324002:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774559.324006:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774559.324010:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774559.580510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774559.580516:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774559.581207:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774559.581209:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774559.581386:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774559.581387:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774559.581597:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774559.581598:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774559.581946:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774559.581948:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775774560.347981:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774560.347984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774560.347987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774560.859888:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310948597, at 4310948608 00000100:00000010:1.0:1775774560.859903:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000020:00000001:1.0:1775774561.371990:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774561.371994:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774561.371996:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774562.396035:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774562.396039:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774562.396043:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774563.420080:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774563.420084:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774563.420088:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774564.443953:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774564.443957:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774564.443961:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774564.699877:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774564.699885:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774564.699888:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774564.699891:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774564.699895:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774564.699900:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16280 00000400:00000200:1.0:1775774564.699909:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774564.699914:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774564.699916:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774564.699918:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.699921:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.699923:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.699925:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774564.699928:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774564.699930:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774564.699932:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774564.699934:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774564.699935:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774564.699944:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774564.699946:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.699947:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.699951:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.699953:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774564.699955:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774564.699957:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774564.699959:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774564.699962:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774564.699964:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774564.699966:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.699968:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774564.699969:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774564.699978:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774564.699980:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774564.699983:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774564.699985:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.699987:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774564.699990:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774564.699993:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774564.699999:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774564.700004:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774564.700011:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099516672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774564.700018:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774564.700022:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774564.700025:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099516672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774564.700043:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774564.700046:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774564.700048:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774564.700049:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774564.700050:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774564.700052:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774564.700054:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16280 00000400:00000200:1.0:1775774564.700056:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774564.700057:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774564.700059:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774564.700061:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700063:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.700065:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700066:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774564.700068:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774564.700069:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774564.700071:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774564.700072:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774564.700074:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774564.700075:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774564.700078:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700079:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.700081:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700083:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774564.700086:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774564.700089:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700092:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774564.700095:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774564.700099:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774564.700103:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700107:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774564.700111:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774564.700115:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700119:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774564.700123:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774564.700126:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774564.700129:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774564.700132:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700134:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774564.700137:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774564.700139:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774564.700143:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774564.700148:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774564.700157:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099516800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774564.700174:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774564.700177:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774564.700184:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099516800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774564.700223:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774564.700234:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774564.700244:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774564.700248:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774564.700252:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774564.700256:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774564.700261:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16280 00000400:00000200:1.0:1775774564.700269:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774564.700305:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774564.700311:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774564.700319:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700326:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.700335:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700340:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774564.700362:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774564.700365:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774564.700368:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774564.700373:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774564.700378:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774564.700383:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774564.700391:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700394:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.700397:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700402:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774564.700407:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774564.700414:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700418:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774564.700421:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774564.700424:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774564.700427:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700428:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774564.700430:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774564.700434:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700438:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774564.700442:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774564.700463:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774564.700467:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774564.700468:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700470:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774564.700471:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774564.700475:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775774564.700475:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000010:1.0:1775774564.700483:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000800:00000200:3.1:1775774564.700483:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1775774564.700491:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774564.700501:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099516928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774564.700517:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774564.700523:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774564.700529:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099516928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774564.700543:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774564.700549:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774564.700583:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774564.700601:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774564.700603:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774564.700608:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774564.700615:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16280 00000400:00000200:1.0:1775774564.700625:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774564.700633:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774564.700638:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774564.700643:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700645:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.700649:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700653:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774564.700658:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774564.700659:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774564.700662:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774564.700664:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774564.700666:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774564.700672:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774564.700674:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700676:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.700680:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700682:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774564.700684:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774564.700690:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700695:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774564.700696:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774564.700700:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774564.700704:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700708:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774564.700710:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774564.700714:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700715:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774564.700721:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774564.700725:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774564.700727:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774564.700728:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700731:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774564.700733:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774564.700736:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774564.700742:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774564.700748:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774564.700756:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099517056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774564.700771:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774564.700775:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774564.700780:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099517056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774564.700786:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774564.700788:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774564.700791:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774564.700792:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774564.700823:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774564.700828:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774564.700833:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16280 00000400:00000200:1.0:1775774564.700843:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774564.700849:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774564.700852:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774564.700857:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700860:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.700863:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700866:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774564.700871:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774564.700873:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774564.700876:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774564.700878:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774564.700882:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774564.700886:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774564.700892:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700895:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774564.700899:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700903:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774564.700906:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774564.700913:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774564.700919:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774564.700924:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774564.700929:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774564.700934:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700937:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774564.700941:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774564.700946:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700951:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774564.700958:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774564.700963:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774564.700967:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774564.700970:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774564.700972:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774564.700975:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774564.700979:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774564.700986:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774564.700992:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774564.701002:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099517184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774564.701021:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774564.701025:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774564.701033:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099517184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774564.701056:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774564.701064:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16290) 00000800:00000200:3.1:1775774564.701120:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774564.701122:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774564.701422:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774564.701424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774564.701728:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774564.701730:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774564.702026:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774564.702029:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775774565.467975:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774565.467979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774565.467982:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774565.979868:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310953717, at 4310953728 00000100:00000010:1.0:1775774565.979880:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000020:00000001:1.0:1775774566.491993:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774566.491996:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774566.491999:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774567.515997:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774567.516001:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774567.516003:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774568.540001:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774568.540004:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774568.540007:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774569.563952:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774569.563955:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774569.563958:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774569.819922:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774569.819930:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774569.819933:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774569.819936:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774569.819939:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774569.819945:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16285 00000400:00000200:1.0:1775774569.819953:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774569.819958:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774569.819961:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774569.819963:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.819966:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.819968:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.819970:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774569.819973:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774569.819974:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774569.819976:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774569.819978:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774569.819980:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774569.819981:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774569.819983:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.819984:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.819986:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.819987:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774569.819989:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774569.819992:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774569.819993:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774569.819995:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774569.819998:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774569.820000:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820001:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774569.820003:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774569.820012:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774569.820015:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774569.820018:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774569.820019:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820022:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774569.820024:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774569.820028:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774569.820034:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775774569.820039:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774569.820047:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099517312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774569.820055:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774569.820059:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774569.820062:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099517312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774569.820082:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774569.820084:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774569.820087:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774569.820088:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774569.820089:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774569.820091:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774569.820093:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16285 00000400:00000200:1.0:1775774569.820095:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774569.820097:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774569.820098:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774569.820100:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820102:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.820103:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820104:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774569.820106:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774569.820107:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774569.820108:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774569.820110:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774569.820111:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774569.820113:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774569.820115:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820116:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.820117:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820118:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774569.820119:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774569.820121:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820122:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774569.820123:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774569.820127:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774569.820128:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820129:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774569.820132:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774569.820135:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820138:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774569.820142:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774569.820144:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774569.820147:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774569.820149:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820151:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774569.820152:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774569.820155:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774569.820162:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775774569.820169:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774569.820178:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099517440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774569.820196:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774569.820201:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774569.820212:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099517440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774569.820229:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774569.820235:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774569.820243:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774569.820245:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774569.820246:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774569.820249:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774569.820254:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16285 00000400:00000200:1.0:1775774569.820260:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774569.820264:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774569.820269:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774569.820277:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820282:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.820290:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820295:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774569.820300:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774569.820303:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774569.820307:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774569.820310:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774569.820315:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774569.820319:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774569.820324:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820329:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.820334:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820338:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774569.820344:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774569.820351:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820356:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774569.820361:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774569.820367:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774569.820372:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820375:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774569.820378:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774569.820384:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820389:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774569.820396:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774569.820400:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774569.820407:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774569.820410:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820414:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774569.820416:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774569.820420:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774569.820427:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774569.820433:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774569.820442:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099517568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1775774569.820450:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774569.820456:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775774569.820462:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774569.820467:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774569.820488:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099517568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774569.820502:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774569.820509:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774569.820517:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774569.820519:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774569.820522:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774569.820527:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774569.820533:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16285 00000400:00000200:1.0:1775774569.820542:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774569.820550:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774569.820556:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774569.820562:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820567:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.820574:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820580:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774569.820585:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774569.820588:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774569.820590:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774569.820593:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774569.820599:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774569.820605:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774569.820613:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820618:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.820623:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820628:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774569.820634:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774569.820642:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820648:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774569.820653:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774569.820658:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774569.820662:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820667:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774569.820671:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774569.820678:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820682:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774569.820691:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774569.820696:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774569.820702:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774569.820706:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.820711:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774569.820714:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774569.820718:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774569.820725:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774569.820734:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774569.820746:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099517696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774569.820770:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774569.820776:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774569.820811:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099517696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774569.820838:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774569.820844:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774569.820853:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774569.820857:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774569.820862:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774569.820867:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774569.820874:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16285 00000400:00000200:1.0:1775774569.820885:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774569.820891:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774569.820897:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774569.820906:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820912:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.820919:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820925:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774569.820931:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774569.820935:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774569.820940:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774569.820944:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774569.820950:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774569.820956:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774569.820966:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820973:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774569.820981:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774569.820989:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774569.820996:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774569.821003:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774569.821011:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774569.821016:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774569.821023:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774569.821030:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.821036:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774569.821040:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774569.821048:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.821053:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774569.821063:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774569.821070:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000800:00000200:3.1:1775774569.821074:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775774569.821076:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:3.1:1775774569.821077:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 02000000:00000001:1.0:1775774569.821080:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774569.821085:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774569.821089:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774569.821094:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774569.821102:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774569.821109:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774569.821121:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099517824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774569.821145:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774569.821152:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774569.821162:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099517824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774569.821192:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774569.821202:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16295) 00000800:00000200:3.1:1775774569.821338:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774569.821340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774569.821549:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774569.821550:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774569.821819:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774569.821820:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775774570.587978:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774570.587982:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774570.587986:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774571.099862:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310958837, at 4310958848 00000100:00000010:1.0:1775774571.099874:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c60. 00000020:00000001:1.0:1775774571.611967:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774571.611971:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774571.611973:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774572.635999:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774572.636004:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774572.636008:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774573.660068:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774573.660073:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774573.660077:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774574.684034:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774574.684039:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774574.684043:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774574.940578:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774574.940582:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774574.941144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774574.941146:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774574.941358:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774574.941361:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774574.941568:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774574.941570:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774574.941872:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774574.941874:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775774575.707976:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774575.707980:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774575.707983:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774576.219916:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310963957, at 4310963968 00000100:00000010:1.0:1775774576.219933:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000020:00000001:1.0:1775774576.732038:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774576.732043:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774576.732047:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774577.755971:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774577.755976:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774577.755980:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774578.780028:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774578.780032:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774578.780035:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774579.803971:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774579.803976:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774579.803979:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774580.060497:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774580.060938:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774580.061091:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774580.061245:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774580.062760:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775774580.828008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774580.828011:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774580.828014:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774581.339864:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310969077, at 4310969088 00000100:00000010:1.0:1775774581.339877:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344a80. 00000020:00000001:1.0:1775774581.851977:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774581.851980:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774581.851983:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774582.876002:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774582.876005:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774582.876008:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774583.899945:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774583.899948:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774583.899951:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774584.923995:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774584.923999:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774584.924001:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774585.948006:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774585.948011:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774585.948015:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774586.459894:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310974197, at 4310974208 00000100:00000010:1.0:1775774586.459910:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775774586.972009:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774586.972012:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774586.972015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774587.995972:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774587.995975:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774587.995979:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774589.019981:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774589.019984:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774589.019987:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774590.044033:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774590.044037:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774590.044040:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774590.299896:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774590.299907:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774590.299909:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774590.299913:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774590.299917:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774590.299922:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16305 00000400:00000200:1.0:1775774590.299931:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774590.299936:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774590.299938:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774590.299941:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.299943:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.299945:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.299947:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774590.299950:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774590.299951:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774590.299953:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774590.299956:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774590.299957:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774590.299959:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774590.299961:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.299962:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.299963:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.299965:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774590.299967:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774590.299969:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774590.299971:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774590.299973:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774590.299975:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774590.299977:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.299979:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774590.299981:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774590.299988:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774590.299991:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774590.299993:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774590.299994:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.299997:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774590.299999:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774590.300002:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774590.300009:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774590.300015:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774590.300022:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099519872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774590.300030:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774590.300033:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774590.300036:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099519872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774590.300054:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774590.300057:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774590.300059:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774590.300060:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774590.300061:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774590.300063:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774590.300065:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16305 00000400:00000200:1.0:1775774590.300067:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774590.300069:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774590.300071:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774590.300073:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300074:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.300075:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300077:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774590.300079:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774590.300080:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774590.300087:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774590.300089:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774590.300090:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774590.300092:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774590.300094:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300096:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.300100:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300102:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774590.300103:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774590.300105:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300107:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774590.300108:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774590.300113:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774590.300129:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300131:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774590.300134:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774590.300136:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300137:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774590.300140:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774590.300142:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774590.300143:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774590.300147:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300149:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774590.300155:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774590.300157:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774590.300161:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774590.300167:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774590.300177:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099520000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774590.300244:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774590.300247:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774590.300254:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099520000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774590.300277:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774590.300285:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774590.300291:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774590.300292:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774590.300293:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774590.300296:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774590.300300:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16305 00000400:00000200:1.0:1775774590.300313:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774590.300318:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774590.300320:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774590.300324:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300330:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.300335:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300340:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774590.300347:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774590.300350:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774590.300354:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774590.300357:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774590.300359:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774590.300365:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774590.300374:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300380:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.300387:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300397:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774590.300402:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774590.300409:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300418:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774590.300430:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774590.300439:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774590.300447:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300455:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774590.300461:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774590.300466:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300469:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774590.300472:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774590.300475:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774590.300483:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774590.300486:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300489:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774590.300490:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774590.300494:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774590.300505:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774590.300512:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774590.300525:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099520128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774590.300544:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774590.300547:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774590.300557:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099520128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774590.300593:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774590.300601:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774590.300611:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774590.300614:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774590.300615:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774590.300617:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774590.300619:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16305 00000400:00000200:1.0:1775774590.300634:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774590.300648:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774590.300652:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774590.300655:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300662:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.300666:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300676:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774590.300682:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774590.300684:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774590.300698:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774590.300706:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774590.300713:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774590.300723:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774590.300732:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300738:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.300742:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300744:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774590.300756:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774590.300765:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774590.300768:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774590.300770:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774590.300773:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774590.300810:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300812:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774590.300814:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774590.300817:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300818:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774590.300822:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774590.300824:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774590.300827:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774590.300827:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.300829:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774590.300830:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774590.300832:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774590.300835:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774590.300839:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774590.300846:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099520256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774590.300914:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774590.300922:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774590.300936:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099520256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774590.300988:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774590.300996:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774590.301010:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774590.301013:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774590.301019:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774590.301026:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774590.301029:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16305 00000400:00000200:1.0:1775774590.301037:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774590.301045:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774590.301050:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774590.301057:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.301062:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.301065:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.301066:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774590.301068:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774590.301069:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774590.301075:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774590.301077:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774590.301080:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774590.301082:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774590.301085:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.301088:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774590.301097:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774590.301099:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774590.301102:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774590.301106:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774590.301110:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774590.301128:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774590.301136:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774590.301141:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.301145:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774590.301147:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774590.301161:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.301164:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774590.301172:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774590.301179:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774590.301187:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774590.301191:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774590.301202:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774590.301206:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774590.301210:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774590.301217:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775774590.301225:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774590.301235:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099520384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774590.301252:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774590.301257:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774590.301268:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099520384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774590.301321:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774590.301328:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16315) 00000020:00000001:1.0:1775774591.068006:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774591.068011:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774591.068015:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774591.579885:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310979318, at 4310979328 00000100:00000010:1.0:1775774591.579901:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775774592.092034:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774592.092043:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774592.092059:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774593.116027:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774593.116031:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774593.116034:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774594.140033:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774594.140063:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774594.140067:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774595.163995:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774595.164008:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774595.164011:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774595.420516:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774595.420521:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774595.421085:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774595.421086:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774595.421333:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774595.421334:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774595.421516:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774595.421518:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775774595.421690:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774595.421692:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775774596.187924:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774596.187928:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774596.187931:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774596.699886:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310984437, at 4310984448 00000100:00000010:1.0:1775774596.699904:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000020:00000001:1.0:1775774597.211943:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774597.211946:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774597.211949:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774598.236028:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774598.236032:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774598.236036:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774599.260002:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774599.260007:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774599.260011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774600.284071:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774600.284076:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774600.284081:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774600.540508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774600.540515:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774600.541328:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774600.541331:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774600.541648:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774600.541650:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774600.541998:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774600.542001:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774600.542433:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775774600.542436:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775774601.308080:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774601.308084:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774601.308087:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774601.819910:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310989557, at 4310989568 00000100:00000010:1.0:1775774601.821099:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000020:00000001:1.0:1775774602.332035:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774602.332038:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774602.332041:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774603.356005:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774603.356014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774603.356017:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774604.379963:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774604.379968:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774604.379972:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774605.403940:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774605.403951:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774605.403956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774606.428014:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774606.428026:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774606.428031:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774606.939904:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310994677, at 4310994688 00000100:00000010:1.0:1775774606.939921:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775774607.452007:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774607.452010:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774607.452013:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774608.476048:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774608.476052:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774608.476055:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774609.500040:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774609.500044:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774609.500047:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774610.523955:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774610.523959:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774610.523963:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774611.547983:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774611.547986:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774611.547989:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774612.059871:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4310999797, at 4310999808 00000100:00000010:1.0:1775774612.059885:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775774612.571993:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774612.571998:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774612.572001:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774613.595933:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774613.595936:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774613.595939:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774614.619949:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774614.619951:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774614.619954:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774615.644032:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774615.644035:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774615.644038:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774616.667992:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774616.667997:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774616.668010:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774617.179884:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311004918, at 4311004928 00000100:00000010:1.0:1775774617.179905:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775774617.691999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774617.692002:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774617.692005:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774618.716029:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774618.716034:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774618.716038:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774619.739993:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774619.739998:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774619.740001:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774620.763958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774620.763962:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774620.763965:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774621.019895:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774621.019903:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774621.019905:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774621.019909:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774621.019912:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774621.019917:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16336 00000400:00000200:1.0:1775774621.019926:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774621.019931:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774621.019933:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774621.019936:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.019939:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.019941:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.019943:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774621.019946:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774621.019947:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774621.019950:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774621.019952:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774621.019959:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774621.019961:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774621.019963:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.019964:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.019965:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.019967:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774621.019969:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774621.019971:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774621.019973:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774621.019975:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774621.019977:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774621.019979:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.019981:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774621.019982:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774621.019990:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774621.019993:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774621.019995:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774621.019997:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.019999:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774621.020001:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774621.020004:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774621.020010:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775774621.020016:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774621.020023:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099523712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774621.020031:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774621.020034:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774621.020038:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099523712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774621.020056:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774621.020059:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774621.020062:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774621.020063:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774621.020064:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774621.020065:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774621.020067:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16336 00000400:00000200:1.0:1775774621.020069:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774621.020072:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774621.020073:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774621.020076:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020078:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.020079:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020081:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774621.020083:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774621.020083:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774621.020084:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774621.020086:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774621.020090:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774621.020091:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774621.020095:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020098:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.020102:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020104:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774621.020108:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774621.020112:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020118:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774621.020122:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774621.020127:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774621.020133:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020135:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774621.020140:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774621.020146:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020149:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774621.020156:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774621.020160:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774621.020165:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774621.020169:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020172:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774621.020175:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774621.020180:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774621.020188:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775774621.020195:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774621.020216:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099523840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774621.020237:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774621.020241:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774621.020248:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099523840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774621.020268:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774621.020272:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774621.020276:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774621.020279:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774621.020283:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774621.020285:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774621.020289:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16336 00000400:00000200:1.0:1775774621.020295:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774621.020301:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774621.020303:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774621.020308:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020312:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.020316:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020319:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774621.020322:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774621.020325:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774621.020329:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774621.020332:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774621.020336:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774621.020341:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774621.020346:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020358:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.020365:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020370:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774621.020383:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774621.020390:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020395:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774621.020400:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774621.020404:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774621.020411:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020414:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774621.020419:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774621.020424:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020429:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774621.020446:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774621.020451:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774621.020457:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774621.020460:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020462:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774621.020465:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774621.020469:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774621.020476:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774621.020483:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774621.020493:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099523968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774621.020514:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774621.020519:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774621.020529:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099523968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774621.020544:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774621.020549:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774621.020557:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774621.020559:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774621.020562:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774621.020565:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774621.020571:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16336 00000400:00000200:1.0:1775774621.020577:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774621.020600:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774621.020603:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774621.020610:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020617:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.020623:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020628:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774621.020633:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774621.020637:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774621.020642:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774621.020645:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774621.020650:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774621.020656:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774621.020662:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020666:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.020672:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020677:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774621.020691:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774621.020699:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020703:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774621.020705:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774621.020710:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774621.020716:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020722:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774621.020725:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774621.020731:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020736:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774621.020744:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774621.020749:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774621.020763:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774621.020767:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.020773:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774621.020777:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774621.020818:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774621.020826:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774621.020833:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774621.020843:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099524096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774621.020862:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774621.020866:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774621.020877:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099524096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774621.020914:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774621.020920:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774621.020928:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774621.020931:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774621.020936:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774621.020941:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774621.020947:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16336 00000400:00000200:1.0:1775774621.020957:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774621.020964:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774621.020970:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774621.020978:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020984:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.020989:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.020995:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774621.021001:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774621.021005:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774621.021009:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774621.021013:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774621.021018:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774621.021024:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774621.021033:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.021039:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774621.021047:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774621.021054:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774621.021060:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774621.021068:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774621.021075:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774621.021080:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774621.021086:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774621.021092:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.021097:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774621.021102:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774621.021110:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.021115:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774621.021123:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774621.021128:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774621.021134:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774621.021137:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774621.021142:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774621.021147:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774621.021150:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774621.021157:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774621.021165:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774621.021175:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099524224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774621.021199:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774621.021204:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774621.021215:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099524224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774621.021250:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774621.021259:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16346) 00000020:00000001:1.0:1775774621.787989:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774621.787993:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774621.787997:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774622.299874:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311010037, at 4311010048 00000100:00000010:1.0:1775774622.299889:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775774622.812069:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774622.812083:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774622.812086:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774623.835981:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774623.835985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774623.835988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774624.859942:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774624.859946:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774624.859948:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774625.883978:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774625.883982:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774625.883984:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774626.139853:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774626.139861:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774626.139863:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774626.139866:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774626.139870:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774626.139875:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16341 00000400:00000200:1.0:1775774626.139883:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774626.139888:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774626.139890:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774626.139893:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.139895:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.139897:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.139899:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774626.139902:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774626.139903:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774626.139905:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774626.139907:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774626.139908:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774626.139910:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774626.139912:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.139913:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.139914:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.139915:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774626.139918:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774626.139921:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774626.139922:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774626.139925:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774626.139927:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774626.139929:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.139931:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774626.139932:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774626.139941:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774626.139943:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774626.139946:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774626.139947:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.139949:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774626.139951:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774626.139955:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774626.139962:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774626.139968:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774626.139977:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099524352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774626.139988:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774626.139993:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774626.139998:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099524352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774626.140024:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774626.140028:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774626.140031:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774626.140032:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774626.140033:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774626.140036:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774626.140038:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16341 00000400:00000200:1.0:1775774626.140041:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774626.140043:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774626.140045:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774626.140048:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140049:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.140051:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140053:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774626.140055:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774626.140056:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774626.140059:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774626.140060:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774626.140064:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774626.140067:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774626.140078:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140082:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.140085:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140090:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774626.140092:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774626.140096:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140099:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774626.140101:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774626.140106:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774626.140110:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140114:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774626.140119:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774626.140122:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140124:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774626.140130:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774626.140132:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774626.140134:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774626.140137:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140139:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774626.140141:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774626.140143:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774626.140148:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774626.140152:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774626.140158:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099524480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774626.140165:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774626.140169:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774626.140175:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099524480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774626.140186:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774626.140191:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774626.140196:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774626.140199:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774626.140200:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774626.140203:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774626.140208:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16341 00000400:00000200:1.0:1775774626.140212:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774626.140217:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774626.140219:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774626.140221:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140222:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.140223:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140225:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774626.140228:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774626.140229:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774626.140230:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774626.140231:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774626.140235:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774626.140239:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774626.140242:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140246:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.140250:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140252:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774626.140257:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774626.140263:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140267:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774626.140270:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774626.140273:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774626.140276:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140279:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774626.140281:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774626.140286:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140289:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774626.140293:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775774626.140296:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774626.140301:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774626.140304:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140306:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774626.140309:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774626.140311:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774626.140316:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774626.140319:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774626.140324:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099524608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774626.140336:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774626.140339:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774626.140346:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099524608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774626.140361:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774626.140365:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774626.140369:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774626.140372:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774626.140374:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774626.140377:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774626.140381:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16341 00000400:00000200:1.0:1775774626.140389:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774626.140392:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774626.140394:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774626.140399:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140403:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.140404:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140407:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774626.140411:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774626.140412:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774626.140415:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774626.140417:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774626.140420:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774626.140424:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774626.140429:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140431:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.140436:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140440:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774626.140445:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774626.140450:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140455:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774626.140460:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774626.140464:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774626.140466:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140467:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774626.140471:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774626.140476:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140478:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774626.140484:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774626.140486:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774626.140512:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774626.140513:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140515:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774626.140516:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774626.140517:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774626.140521:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774626.140527:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774626.140531:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099524736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774626.140545:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774626.140548:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774626.140553:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099524736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774626.140583:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774626.140586:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774626.140589:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774626.140592:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774626.140593:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774626.140596:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774626.140599:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16341 00000400:00000200:1.0:1775774626.140606:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774626.140611:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774626.140612:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774626.140617:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140621:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.140624:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140629:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774626.140633:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774626.140636:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774626.140637:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774626.140638:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774626.140641:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774626.140643:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774626.140649:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140652:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774626.140656:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140660:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774626.140665:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774626.140668:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774626.140673:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774626.140678:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774626.140681:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774626.140686:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140689:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774626.140691:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774626.140695:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140699:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774626.140703:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774626.140706:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774626.140711:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774626.140712:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774626.140714:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774626.140717:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774626.140719:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774626.140723:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774626.140727:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774626.140733:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099524864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774626.140750:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774626.140755:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774626.140767:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099524864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774626.140812:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774626.140818:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16351) 00000020:00000001:1.0:1775774626.907959:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774626.907962:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774626.907965:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774627.419859:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311015157, at 4311015168 00000100:00000010:1.0:1775774627.419877:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774627.931964:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774627.931967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774627.931970:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774628.955943:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774628.955947:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774628.955950:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774629.979999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774629.980008:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774629.980011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774631.004002:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774631.004006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774631.004009:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774632.027957:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774632.027963:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774632.027966:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774632.539883:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311020278, at 4311020288 00000100:00000010:1.0:1775774632.539900:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775774633.052021:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774633.052025:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774633.052029:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774634.075973:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774634.075977:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774634.075981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774635.099975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774635.099979:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774635.099984:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774636.124022:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774636.124025:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774636.124027:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774637.147952:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774637.147956:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774637.147958:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774637.659859:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311025397, at 4311025408 00000100:00000010:1.0:1775774637.659878:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775774638.171952:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774638.171955:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774638.171958:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774639.196049:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774639.196053:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774639.196056:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774640.220039:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774640.220042:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774640.220045:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774641.243975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774641.243978:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774641.243981:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774642.267968:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774642.267972:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774642.267976:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774642.779871:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311030517, at 4311030528 00000100:00000010:1.0:1775774642.779884:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775774643.292057:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774643.292061:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774643.292063:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774644.315948:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774644.315951:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774644.315954:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774645.339977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774645.339980:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774645.339983:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774646.364003:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774646.364005:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774646.364009:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774647.388001:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774647.388005:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774647.388008:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774647.899907:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311035637, at 4311035648 00000100:00000010:1.0:1775774647.899925:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775774648.411981:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774648.411985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774648.411988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774649.435975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774649.435980:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774649.435984:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774650.459977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774650.459981:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774650.459985:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774651.483983:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774651.483986:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774651.483989:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774652.507924:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774652.507928:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774652.507932:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774653.019892:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311040758, at 4311040768 00000100:00000010:1.0:1775774653.019904:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775774653.532063:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774653.532067:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774653.532069:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774654.555949:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774654.555952:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774654.555954:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774655.580004:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774655.580008:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774655.580012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774656.604008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774656.604013:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774656.604017:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774656.859884:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774656.859892:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774656.859895:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774656.859898:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774656.859901:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774656.859907:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16372 00000400:00000200:1.0:1775774656.859915:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774656.859921:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774656.859923:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774656.859925:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.859928:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.859930:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.859932:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774656.859936:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774656.859937:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774656.859939:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774656.859941:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774656.859943:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774656.859944:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774656.859946:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.859947:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.859949:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.859950:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774656.859952:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774656.859967:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774656.859969:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774656.859972:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774656.859975:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774656.859977:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.859979:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774656.859981:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774656.859989:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774656.859992:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774656.859995:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774656.859996:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.859999:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774656.860001:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774656.860004:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774656.860012:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774656.860017:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774656.860024:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099528192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774656.860032:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774656.860036:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774656.860040:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099528192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774656.860069:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774656.860073:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774656.860076:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774656.860078:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774656.860080:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774656.860083:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774656.860086:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16372 00000400:00000200:1.0:1775774656.860091:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774656.860095:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774656.860098:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774656.860102:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860106:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.860112:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860117:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774656.860122:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774656.860126:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774656.860130:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774656.860141:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774656.860144:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774656.860147:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774656.860154:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860161:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.860166:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860170:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774656.860174:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774656.860179:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860182:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774656.860187:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774656.860195:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774656.860200:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860204:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774656.860209:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774656.860214:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860218:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774656.860225:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774656.860231:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774656.860238:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774656.860242:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860245:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774656.860247:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774656.860251:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774656.860258:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775774656.860263:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774656.860271:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099528320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774656.860283:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774656.860287:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774656.860293:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099528320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774656.860305:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774656.860310:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774656.860317:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774656.860321:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774656.860325:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774656.860329:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774656.860337:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16372 00000400:00000200:1.0:1775774656.860350:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774656.860357:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774656.860360:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774656.860368:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860372:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.860377:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860383:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774656.860388:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774656.860390:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774656.860393:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774656.860396:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774656.860399:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774656.860400:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774656.860404:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860409:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.860413:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860415:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774656.860418:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774656.860431:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860437:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774656.860440:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774656.860443:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774656.860446:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860451:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774656.860455:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774656.860460:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860464:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774656.860469:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775774656.860474:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774656.860479:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774656.860482:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860486:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774656.860489:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774656.860493:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774656.860498:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775774656.860504:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774656.860515:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099528448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774656.860527:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774656.860531:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774656.860539:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099528448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774656.860550:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774656.860553:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774656.860558:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774656.860560:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774656.860563:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774656.860568:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774656.860573:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16372 00000400:00000200:1.0:1775774656.860581:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774656.860588:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774656.860591:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774656.860596:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860599:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.860603:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860607:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774656.860612:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774656.860614:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774656.860617:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774656.860620:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774656.860623:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774656.860627:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774656.860634:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860639:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.860643:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860649:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774656.860654:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774656.860659:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860663:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774656.860667:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774656.860670:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774656.860675:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860680:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774656.860684:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774656.860691:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860697:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774656.860705:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774656.860709:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774656.860712:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774656.860715:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860721:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774656.860724:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774656.860727:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774656.860735:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774656.860742:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774656.860753:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099528576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774656.860769:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774656.860772:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774656.860815:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099528576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774656.860836:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774656.860841:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774656.860846:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774656.860848:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774656.860851:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774656.860856:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774656.860860:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16372 00000400:00000200:1.0:1775774656.860866:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774656.860870:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774656.860873:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774656.860877:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860879:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.860884:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860888:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774656.860892:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774656.860895:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774656.860897:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774656.860899:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774656.860902:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774656.860906:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774656.860911:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860914:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774656.860918:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860921:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774656.860925:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774656.860930:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774656.860936:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774656.860939:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774656.860942:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774656.860946:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860948:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774656.860950:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774656.860951:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860952:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774656.860973:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774656.860977:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774656.860980:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774656.860982:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774656.860985:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774656.860987:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774656.860989:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774656.860994:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774656.860999:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774656.861006:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099528704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774656.861017:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774656.861021:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774656.861025:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099528704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774656.861036:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774656.861042:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16382) 00000020:00000001:1.0:1775774657.628042:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774657.628046:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774657.628049:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774658.139895:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311045877, at 4311045888 00000100:00000010:1.0:1775774658.139915:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775774658.652052:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774658.652056:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774658.652059:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774659.675969:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774659.675973:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774659.675977:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774660.700026:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774660.700029:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774660.700032:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774661.723965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774661.723968:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774661.723971:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774662.748035:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774662.748039:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774662.748041:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774663.259898:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311050997, at 4311051008 00000100:00000010:1.0:1775774663.259912:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774663.772047:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774663.772052:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774663.772075:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774664.796051:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774664.796057:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774664.796061:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774665.819950:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774665.819954:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774665.819956:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774666.843947:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774666.843950:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774666.843953:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774667.868012:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774667.868016:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774667.868019:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774668.379874:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311056117, at 4311056128 00000100:00000010:1.0:1775774668.379891:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775774668.892049:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774668.892052:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774668.892056:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774669.915964:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774669.915969:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774669.915972:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774670.939964:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774670.939969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774670.939973:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774671.963962:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774671.963967:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774671.963971:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774672.988017:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774672.988020:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774672.988023:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774673.499931:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311061237, at 4311061248 00000100:00000010:1.0:1775774673.499948:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775774674.012005:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774674.012010:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774674.012014:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774675.036003:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774675.036008:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774675.036013:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774676.059962:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774676.059965:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774676.059967:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774677.083969:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774677.083972:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774677.083975:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774678.107986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774678.107989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774678.107993:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774678.619855:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311066357, at 4311066368 00000100:00000010:1.0:1775774678.619872:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775774679.131965:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774679.131970:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774679.131974:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774680.156035:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774680.156039:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774680.156042:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774681.179983:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774681.179987:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774681.179990:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774682.203987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774682.203991:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774682.203994:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774683.227981:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774683.227984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774683.227987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774683.739879:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311071477, at 4311071488 00000100:00000010:1.0:1775774683.739891:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775774684.251975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774684.251978:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774684.251982:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774685.276014:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774685.276019:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774685.276023:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774686.300004:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774686.300008:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774686.300011:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774687.323965:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774687.323970:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774687.323973:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774688.348000:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774688.348007:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774688.348012:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774688.859914:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311076597, at 4311076608 00000100:00000010:1.0:1775774688.859938:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774689.372000:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774689.372006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774689.372010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774690.396013:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774690.396019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774690.396023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774691.419944:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774691.419949:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774691.419954:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774692.443945:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774692.443948:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774692.443951:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774693.467997:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774693.468000:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774693.468003:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774693.979870:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311081717, at 4311081728 00000100:00000010:1.0:1775774693.979888:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775774694.491952:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774694.491955:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774694.491958:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774695.516013:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774695.516019:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774695.516023:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774696.539974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774696.539977:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774696.539980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774697.563931:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774697.563934:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774697.563937:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774698.587980:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774698.587984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774698.587987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774699.099860:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311086837, at 4311086848 00000100:00000010:1.0:1775774699.099874:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775774699.611982:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774699.611987:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774699.611990:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774700.636057:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774700.636060:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774700.636063:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774701.660009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774701.660017:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774701.660022:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774702.684034:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774702.684037:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774702.684041:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774702.939915:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774702.939926:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774702.939928:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774702.939932:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774702.939936:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774702.939944:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16418 00000400:00000200:1.0:1775774702.939954:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774702.939961:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774702.939965:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774702.939969:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.939972:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.939974:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.939977:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774702.939981:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774702.939983:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774702.939986:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774702.939989:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774702.939991:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774702.939994:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774702.939998:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940000:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940002:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940004:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774702.940007:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774702.940010:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940013:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774702.940017:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774702.940020:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774702.940024:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940026:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774702.940028:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774702.940037:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774702.940041:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774702.940044:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774702.940046:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940049:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774702.940052:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774702.940056:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774702.940065:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774702.940071:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774702.940081:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099533952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774702.940101:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774702.940105:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774702.940111:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099533952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774702.940138:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774702.940141:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774702.940145:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774702.940146:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774702.940148:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774702.940163:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774702.940166:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16418 00000400:00000200:1.0:1775774702.940170:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774702.940174:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774702.940178:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774702.940181:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940184:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940186:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940190:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774702.940195:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774702.940196:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774702.940200:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774702.940204:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774702.940208:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774702.940213:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774702.940223:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940230:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940235:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940241:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774702.940244:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774702.940249:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940255:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774702.940260:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774702.940267:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774702.940271:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940273:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774702.940278:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774702.940281:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940286:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774702.940290:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774702.940293:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774702.940296:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774702.940298:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940301:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774702.940302:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774702.940306:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774702.940318:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774702.940324:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774702.940334:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099534080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774702.940351:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774702.940357:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774702.940366:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099534080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774702.940382:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774702.940387:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774702.940390:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774702.940391:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774702.940393:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774702.940398:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774702.940403:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16418 00000400:00000200:1.0:1775774702.940408:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774702.940414:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774702.940419:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774702.940425:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940428:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940433:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940436:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774702.940441:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774702.940444:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774702.940447:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774702.940450:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774702.940452:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774702.940456:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774702.940463:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940467:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940472:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940476:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774702.940481:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774702.940484:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940488:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774702.940490:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774702.940493:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774702.940497:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940499:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774702.940502:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774702.940507:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940510:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774702.940513:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774702.940518:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774702.940523:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774702.940526:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940528:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774702.940529:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774702.940531:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774702.940536:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774702.940542:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774702.940550:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099534208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774702.940558:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774702.940562:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774702.940571:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099534208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774702.940582:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774702.940586:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774702.940592:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774702.940595:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774702.940597:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774702.940600:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774702.940604:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16418 00000400:00000200:1.0:1775774702.940610:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774702.940614:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774702.940616:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774702.940620:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940623:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940626:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940631:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774702.940632:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774702.940633:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774702.940635:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774702.940637:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774702.940639:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774702.940641:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774702.940644:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940646:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940650:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940651:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774702.940656:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774702.940658:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940663:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774702.940666:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774702.940669:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774702.940672:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940677:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774702.940679:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774702.940682:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940685:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774702.940688:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774702.940692:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774702.940696:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774702.940698:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940702:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774702.940704:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774702.940706:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774702.940710:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774702.940714:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774702.940723:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099534336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774702.940732:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774702.940736:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774702.940741:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099534336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774702.940753:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774702.940755:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774702.940758:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774702.940760:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774702.940763:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774702.940767:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774702.940772:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16418 00000400:00000200:1.0:1775774702.940802:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774702.940804:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774702.940806:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774702.940808:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940810:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940811:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940813:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774702.940815:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774702.940815:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774702.940816:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774702.940819:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774702.940821:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774702.940824:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774702.940831:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940835:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774702.940836:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940838:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774702.940839:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774702.940841:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774702.940846:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774702.940847:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774702.940849:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774702.940853:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940854:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774702.940859:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774702.940864:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940866:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774702.940870:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774702.940873:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774702.940877:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774702.940878:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774702.940882:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774702.940884:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774702.940887:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774702.940892:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774702.940895:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774702.940903:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099534464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774702.940917:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774702.940921:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774702.940928:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099534464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774702.940942:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774702.940948:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16428) 00000020:00000001:1.0:1775774703.708008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774703.708012:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774703.708015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774704.219858:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311091957, at 4311091968 00000100:00000010:1.0:1775774704.219880:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774704.731988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774704.731994:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774704.732003:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774705.755983:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774705.755989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774705.755994:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774706.779993:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774706.779996:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774706.779999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774707.804020:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774707.804027:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774707.804033:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774708.059897:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774708.059909:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774708.059912:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774708.059917:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774708.059921:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774708.059929:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16423 00000400:00000200:1.0:1775774708.059940:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774708.059946:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774708.059949:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774708.059952:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.059955:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.059957:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.059960:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774708.059964:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774708.059965:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774708.059968:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774708.059971:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774708.059972:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774708.059974:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774708.059976:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.059977:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.059979:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.059980:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774708.059983:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774708.059986:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774708.059987:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774708.059990:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774708.059993:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774708.059995:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.059997:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774708.059999:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774708.060010:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774708.060013:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774708.060017:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774708.060018:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060020:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774708.060023:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774708.060028:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774708.060036:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775774708.060043:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774708.060052:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099534592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774708.060076:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774708.060081:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774708.060085:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099534592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774708.060106:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774708.060109:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774708.060112:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774708.060114:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774708.060115:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774708.060117:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774708.060119:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16423 00000400:00000200:1.0:1775774708.060122:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774708.060125:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774708.060127:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774708.060130:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060132:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.060134:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060136:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774708.060137:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774708.060138:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774708.060139:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774708.060141:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774708.060142:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774708.060144:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774708.060147:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060153:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.060155:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060158:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774708.060167:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774708.060170:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060172:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774708.060178:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774708.060184:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774708.060191:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060196:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774708.060202:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774708.060211:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060214:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774708.060219:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774708.060226:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774708.060229:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774708.060230:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060232:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774708.060236:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774708.060238:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774708.060242:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775774708.060247:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774708.060253:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099534720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774708.060274:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774708.060279:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774708.060284:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099534720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774708.060305:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774708.060307:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774708.060311:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774708.060313:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774708.060315:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774708.060320:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774708.060324:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16423 00000400:00000200:1.0:1775774708.060330:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774708.060332:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774708.060333:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774708.060339:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060341:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.060345:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060347:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774708.060350:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774708.060353:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774708.060355:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774708.060356:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774708.060358:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774708.060359:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774708.060364:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060367:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.060369:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060371:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774708.060373:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774708.060379:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060383:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774708.060385:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774708.060389:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774708.060394:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060397:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774708.060398:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774708.060403:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060406:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774708.060411:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774708.060415:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774708.060422:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774708.060425:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060431:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774708.060435:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774708.060440:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774708.060445:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774708.060452:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774708.060460:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099534848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774708.060471:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774708.060474:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774708.060483:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099534848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774708.060498:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774708.060500:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774708.060503:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774708.060504:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774708.060505:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774708.060509:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774708.060514:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16423 00000400:00000200:1.0:1775774708.060518:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774708.060521:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774708.060523:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774708.060532:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060535:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.060537:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060539:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774708.060541:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774708.060544:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774708.060545:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774708.060550:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774708.060556:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774708.060561:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774708.060567:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060575:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.060582:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060584:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774708.060588:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774708.060594:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060600:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774708.060603:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774708.060606:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774708.060610:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060616:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774708.060619:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774708.060622:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060623:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774708.060626:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774708.060629:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774708.060634:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774708.060636:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060639:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774708.060641:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774708.060645:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774708.060655:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774708.060658:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774708.060667:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099534976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774708.060675:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774708.060678:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774708.060685:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099534976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774708.060700:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774708.060702:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774708.060705:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774708.060705:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774708.060706:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774708.060709:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774708.060712:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16423 00000400:00000200:1.0:1775774708.060721:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774708.060727:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774708.060732:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774708.060734:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060735:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.060736:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060737:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774708.060739:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774708.060743:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774708.060749:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774708.060750:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774708.060754:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774708.060761:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774708.060768:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060773:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774708.060806:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060809:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774708.060812:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774708.060815:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774708.060817:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774708.060820:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774708.060828:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774708.060834:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060838:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774708.060841:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774708.060850:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060854:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774708.060859:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774708.060862:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774708.060866:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774708.060868:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774708.060874:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774708.060876:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774708.060878:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774708.060882:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774708.060887:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774708.060893:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099535104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774708.060943:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774708.060945:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774708.060962:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099535104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774708.061028:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774708.061033:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16433) 00000800:00000200:2.1:1775774708.062237:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775774708.827957:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774708.827964:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774708.827970:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774709.339899:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311097078, at 4311097088 00000100:00000010:1.0:1775774709.339919:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774709.852071:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774709.852074:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774709.852077:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774710.875954:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774710.875958:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774710.875960:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774711.900022:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774711.900026:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774711.900030:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774712.924071:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774712.924077:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774712.924081:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774713.179889:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774713.179900:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774713.179904:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774713.179908:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774713.179913:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774713.179922:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16428 00000400:00000200:1.0:1775774713.179933:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774713.179940:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774713.179944:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774713.179948:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.179953:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.179956:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.179960:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774713.179965:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774713.179968:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774713.179983:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774713.179987:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774713.179990:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774713.179993:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774713.179996:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.179998:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180000:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180002:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774713.180005:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774713.180009:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180012:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774713.180016:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774713.180019:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774713.180022:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180025:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774713.180028:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774713.180041:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774713.180046:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774713.180052:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774713.180054:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180058:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774713.180062:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774713.180068:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774713.180088:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774713.180096:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774713.180108:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099535232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774713.180118:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774713.180123:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774713.180127:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099535232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774713.180152:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774713.180156:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774713.180160:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774713.180162:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774713.180164:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774713.180167:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774713.180171:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16428 00000400:00000200:1.0:1775774713.180177:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774713.180182:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774713.180186:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774713.180193:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180197:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180203:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180209:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774713.180215:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774713.180216:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774713.180219:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774713.180223:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774713.180225:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774713.180229:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774713.180236:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180238:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180243:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180248:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774713.180250:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774713.180253:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180257:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774713.180262:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774713.180268:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774713.180271:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180274:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774713.180279:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774713.180284:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180287:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774713.180294:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774713.180296:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774713.180298:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774713.180301:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180303:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774713.180304:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774713.180306:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774713.180311:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774713.180318:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774713.180325:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099535360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774713.180336:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774713.180339:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774713.180345:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099535360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774713.180357:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774713.180362:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774713.180365:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774713.180367:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774713.180368:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774713.180372:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774713.180376:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16428 00000400:00000200:1.0:1775774713.180381:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774713.180387:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774713.180390:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774713.180395:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180398:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180401:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180404:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774713.180408:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774713.180410:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774713.180414:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774713.180417:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774713.180421:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774713.180424:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774713.180430:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180432:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180434:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180436:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774713.180440:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774713.180453:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180460:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774713.180464:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774713.180467:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774713.180469:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774713.180470:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775774713.180472:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774713.180475:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774713.180478:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180479:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774713.180482:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774713.180484:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774713.180485:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774713.180486:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180488:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774713.180489:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774713.180490:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774713.180494:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774713.180499:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774713.180505:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099535488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774713.180514:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774713.180517:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774713.180519:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099535488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774713.180527:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774713.180529:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774713.180532:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774713.180534:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774713.180535:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774713.180537:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774713.180540:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16428 00000400:00000200:1.0:1775774713.180545:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774713.180550:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774713.180552:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774713.180557:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180559:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180563:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180567:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774713.180571:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774713.180573:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774713.180575:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774713.180578:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774713.180580:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774713.180585:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774713.180587:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180588:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180589:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180591:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774713.180592:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774713.180594:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180598:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774713.180603:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774713.180606:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774713.180609:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180611:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774713.180614:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774713.180617:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180619:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774713.180624:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774713.180627:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774713.180631:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774713.180633:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180637:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774713.180639:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774713.180641:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774713.180646:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774713.180650:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774713.180656:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099535616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774713.180667:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774713.180669:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774713.180674:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099535616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774713.180687:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774713.180690:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774713.180693:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774713.180696:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774713.180697:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774713.180699:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774713.180702:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16428 00000400:00000200:1.0:1775774713.180707:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774713.180712:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774713.180717:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774713.180722:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180724:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180727:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180729:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774713.180735:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774713.180737:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774713.180741:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774713.180742:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774713.180744:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774713.180748:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774713.180756:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180761:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774713.180765:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180770:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774713.180773:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774713.180805:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774713.180809:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774713.180810:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774713.180814:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774713.180817:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180820:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774713.180823:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774713.180827:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180830:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774713.180835:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774713.180839:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774713.180845:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774713.180848:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774713.180852:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774713.180854:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774713.180857:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774713.180861:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774713.180866:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774713.180874:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099535744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774713.180886:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774713.180891:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774713.180897:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099535744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775774713.180903:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775774713.180911:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774713.180914:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16438) 00000800:00000200:2.1:1775774713.181045:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774713.181149:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774713.181252:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775774713.948021:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774713.948024:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774713.948028:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774714.459928:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311102197, at 4311102208 00000100:00000010:1.0:1775774714.459945:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775774714.972030:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774714.972035:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774714.972039:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774715.996055:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774715.996062:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774715.996068:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774717.020052:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774717.020055:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774717.020059:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774718.044008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774718.044014:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774718.044020:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774718.299917:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774718.299926:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774718.299929:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774718.299932:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774718.299937:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774718.299945:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16433 00000400:00000200:1.0:1775774718.299954:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774718.299961:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774718.299964:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774718.299967:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.299969:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.299972:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.299973:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774718.299976:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774718.299979:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774718.299981:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774718.299984:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774718.299986:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774718.299988:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774718.299990:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.299991:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.299993:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.299996:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774718.299999:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774718.300002:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300004:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774718.300006:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774718.300009:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774718.300011:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300013:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774718.300015:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774718.300023:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774718.300025:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774718.300028:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774718.300029:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300032:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774718.300034:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774718.300038:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774718.300044:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775774718.300049:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774718.300056:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099535872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774718.300064:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774718.300068:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774718.300072:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099535872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774718.300095:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774718.300098:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774718.300100:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774718.300101:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774718.300102:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774718.300105:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774718.300106:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16433 00000400:00000200:1.0:1775774718.300108:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774718.300110:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774718.300112:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774718.300114:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300115:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.300117:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300118:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774718.300119:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774718.300121:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774718.300122:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774718.300124:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774718.300127:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774718.300130:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774718.300133:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300136:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.300138:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300140:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774718.300143:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774718.300149:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300158:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774718.300163:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774718.300171:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774718.300175:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300179:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774718.300186:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774718.300189:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300195:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774718.300200:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774718.300205:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774718.300211:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774718.300214:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300217:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774718.300219:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774718.300225:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774718.300235:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774718.300244:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774718.300254:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099536000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774718.300266:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774718.300270:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774718.300291:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099536000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774718.300320:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774718.300323:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774718.300332:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774718.300335:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774718.300338:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774718.300343:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774718.300350:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16433 00000400:00000200:1.0:1775774718.300359:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774718.300366:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774718.300371:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774718.300377:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300380:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.300385:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300389:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774718.300393:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774718.300396:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774718.300398:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774718.300402:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774718.300406:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774718.300412:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774718.300420:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300425:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.300431:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300435:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774718.300441:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774718.300444:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300446:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774718.300448:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774718.300452:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774718.300457:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300462:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774718.300465:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774718.300470:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300474:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774718.300479:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774718.300483:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774718.300489:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774718.300492:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300497:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774718.300500:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774718.300502:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774718.300509:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775774718.300512:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774718.300519:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099536128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775774718.300524:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774718.300531:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000001:1.0:1775774718.300536:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774718.300541:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774718.300550:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099536128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774718.300563:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774718.300567:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774718.300574:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774718.300579:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774718.300582:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774718.300585:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774718.300591:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16433 00000400:00000200:1.0:1775774718.300600:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774718.300605:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774718.300611:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774718.300619:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300625:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.300630:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300633:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774718.300637:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774718.300640:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774718.300642:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774718.300645:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774718.300657:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774718.300662:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774718.300670:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300674:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.300679:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300683:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774718.300689:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774718.300692:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300698:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774718.300701:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774718.300704:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774718.300707:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300710:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774718.300713:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774718.300716:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300719:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774718.300723:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774718.300727:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774718.300732:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774718.300734:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.300738:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774718.300740:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774718.300744:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774718.300752:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774718.300758:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774718.300832:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099536256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774718.300842:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774718.300846:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774718.300853:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099536256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774718.300867:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774718.300875:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774718.300893:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774718.300896:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774718.300898:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774718.300904:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774718.300910:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16433 00000400:00000200:1.0:1775774718.300917:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774718.300925:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774718.300931:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774718.300938:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300943:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.300947:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300951:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774718.300956:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774718.300959:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774718.300962:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774718.300966:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774718.300972:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774718.300977:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774718.300985:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300988:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774718.300992:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774718.300996:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774718.301002:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774718.301011:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774718.301016:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774718.301020:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774718.301025:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774718.301032:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.301035:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774718.301038:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774718.301045:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.301050:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774718.301055:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774718.301061:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774718.301068:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774718.301071:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774718.301074:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774718.301077:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774718.301080:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774718.301087:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774718.301093:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774718.301119:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099536384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774718.301134:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774718.301140:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774718.301149:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099536384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774718.301164:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774718.301171:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16443) 00000800:00000200:2.1:1775774718.301344:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774718.301347:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774718.301656:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774718.301658:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774718.301986:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774718.301989:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774718.302278:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774718.302281:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775774719.067955:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774719.067959:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774719.067963:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774719.579889:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311107317, at 4311107328 00000100:00000010:1.0:1775774719.579906:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775774720.091947:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774720.091950:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774720.091953:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774721.115994:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774721.115998:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774721.116001:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774722.139948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774722.139953:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774722.139957:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774723.164008:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774723.164014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774723.164018:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774723.419960:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774723.419975:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774723.419979:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774723.419985:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774723.419990:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774723.420000:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16438 00000400:00000200:1.0:1775774723.420014:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774723.420022:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774723.420026:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774723.420030:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420035:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.420038:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420041:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774723.420045:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774723.420048:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774723.420051:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774723.420055:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774723.420057:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774723.420060:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774723.420063:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420064:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.420067:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420069:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774723.420072:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774723.420076:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420079:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774723.420082:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774723.420087:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774723.420090:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.420093:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774723.420096:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774723.420109:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774723.420114:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774723.420119:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774723.420121:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.420125:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774723.420128:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774723.420134:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774723.420145:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774723.420154:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774723.420167:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099536512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774723.420181:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774723.420187:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774723.420193:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099536512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774723.420222:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774723.420248:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774723.420253:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774723.420255:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774723.420258:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774723.420262:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774723.420265:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16438 00000400:00000200:1.0:1775774723.420272:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774723.420277:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774723.420281:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774723.420286:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420290:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.420293:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420296:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774723.420300:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774723.420303:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774723.420307:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774723.420312:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774723.420318:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774723.420322:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774723.420331:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420333:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.420345:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420359:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774723.420372:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774723.420383:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420390:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774723.420411:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774723.420423:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774723.420437:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.420442:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774723.420451:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774723.420461:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.420464:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774723.420470:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774723.420474:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774723.420478:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774723.420483:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.420487:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774723.420490:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774723.420498:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774723.420524:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774723.420532:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774723.420540:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099536640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774723.420555:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774723.420563:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774723.420587:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099536640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774723.420600:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774723.420606:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774723.420622:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774723.420626:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774723.420629:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774723.420638:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774723.420644:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16438 00000400:00000200:1.0:1775774723.420652:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774723.420656:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774723.420664:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774723.420671:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420674:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.420715:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420726:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774723.420730:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774723.420738:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774723.420749:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774723.420753:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774723.420779:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774723.420786:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774723.420852:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420858:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.420868:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420876:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774723.420885:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774723.420900:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774723.420910:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774723.420917:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774723.420936:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774723.420946:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.420951:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774723.420956:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774723.420965:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.420971:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774723.420974:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000010:1.0:1775774723.420979:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000800:00000200:2.1:1775774723.420980:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000040:1.0:1775774723.420986:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774723.420994:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774723.420995:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.420997:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774723.420998:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774723.421000:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774723.421015:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774723.421019:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774723.421027:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099536768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774723.421039:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774723.421044:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774723.421051:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099536768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774723.421101:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774723.421110:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774723.421139:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774723.421146:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774723.421149:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774723.421154:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774723.421161:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16438 00000400:00000200:1.0:1775774723.421175:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774723.421190:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774723.421198:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774723.421207:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421217:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.421227:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421231:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774723.421236:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774723.421243:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774723.421247:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774723.421254:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774723.421259:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774723.421266:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774723.421272:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421284:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.421287:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421290:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774723.421296:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774723.421301:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421304:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774723.421308:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774723.421314:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774723.421325:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.421332:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774723.421338:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774723.421348:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.421353:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774723.421361:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774723.421372:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774723.421379:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774723.421383:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.421391:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774723.421397:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774723.421401:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774723.421415:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774723.421421:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774723.421431:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099536896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774723.421483:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774723.421489:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774723.421499:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099536896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774723.421541:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774723.421548:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774723.421561:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774723.421566:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774723.421571:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774723.421578:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774723.421584:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16438 00000400:00000200:1.0:1775774723.421590:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774723.421594:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774723.421599:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774723.421603:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421606:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.421613:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421616:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774723.421621:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774723.421624:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774723.421629:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774723.421633:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774723.421642:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774723.421653:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774723.421670:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421672:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774723.421678:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421685:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774723.421713:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774723.421723:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774723.421735:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774723.421745:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774723.421752:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774723.421759:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.421767:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774723.421776:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774723.421788:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.421861:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774723.421871:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774723.421879:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774723.421886:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774723.421889:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774723.421893:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774723.421898:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774723.421915:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774723.421922:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774723.421933:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774723.421948:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099537024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774723.421975:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774723.421979:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774723.421988:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099537024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774723.422013:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774723.422020:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16448) 00000800:00000200:2.1:1775774723.422171:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774723.422174:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774723.422555:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774723.422557:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774723.422868:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774723.422870:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774723.423202:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774723.423203:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775774724.187982:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774724.187986:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774724.187989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774724.699902:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311112438, at 4311112448 00000100:00000010:1.0:1775774724.699928:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774725.212004:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774725.212008:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774725.212015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774726.235966:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774726.235970:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774726.235973:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774727.260072:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774727.260077:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774727.260082:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774728.284052:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774728.284057:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774728.284062:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774728.540395:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774728.540399:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774728.540948:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774728.540952:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774728.541155:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774728.541157:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774728.541336:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774728.541338:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774728.541520:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774728.541522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775774729.308061:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774729.308068:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774729.308074:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774729.819971:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311117557, at 4311117568 00000100:00000010:1.0:1775774729.819996:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775774730.332020:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774730.332028:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774730.332034:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774731.355974:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774731.355982:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774731.355995:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774732.379991:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774732.379995:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774732.379998:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774733.404016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774733.404020:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774733.404023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774733.660666:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774733.660672:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774733.661673:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774733.661675:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774733.661929:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774733.661931:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774733.662177:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774733.662179:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774733.662528:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774733.662531:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1775774734.428010:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774734.428015:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774734.428019:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774734.939931:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311122678, at 4311122688 00000100:00000010:1.0:1775774734.939954:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775774735.452045:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774735.452049:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774735.452052:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774736.475982:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774736.475985:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774736.475988:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774737.499998:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774737.500003:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774737.500008:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774738.523998:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774738.524001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774738.524004:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774738.779958:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774738.779966:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774738.779968:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774738.779972:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774738.779976:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774738.779982:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16454 00000400:00000200:1.0:1775774738.779990:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774738.779996:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774738.779998:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774738.780001:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780003:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780005:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780007:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774738.780010:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774738.780011:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774738.780013:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774738.780015:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774738.780017:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774738.780019:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774738.780021:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780022:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780023:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780024:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774738.780026:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774738.780028:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780030:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774738.780032:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774738.780035:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774738.780037:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780038:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774738.780040:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774738.780048:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774738.780051:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774738.780054:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774738.780055:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780057:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774738.780058:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774738.780063:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774738.780071:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775774738.780076:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774738.780083:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099538432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774738.780091:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774738.780095:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774738.780098:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099538432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774738.780120:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774738.780123:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774738.780126:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774738.780127:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774738.780128:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774738.780130:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774738.780132:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16454 00000400:00000200:1.0:1775774738.780134:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774738.780136:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774738.780138:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774738.780140:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780141:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780143:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780144:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774738.780146:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774738.780146:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774738.780147:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774738.780149:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774738.780150:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774738.780151:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774738.780153:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780154:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780155:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780157:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774738.780158:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774738.780160:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780162:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774738.780163:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774738.780167:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774738.780169:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780170:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774738.780174:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774738.780176:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780178:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774738.780181:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774738.780182:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774738.780184:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774738.780185:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780189:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774738.780190:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774738.780192:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774738.780196:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775774738.780199:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774738.780206:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099538560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774738.780232:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774738.780236:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774738.780242:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099538560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774738.780251:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774738.780255:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774738.780261:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774738.780265:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774738.780266:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774738.780271:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774738.780276:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16454 00000400:00000200:1.0:1775774738.780284:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774738.780290:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774738.780293:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774738.780299:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780303:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780308:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780314:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774738.780318:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774738.780321:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774738.780324:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774738.780326:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774738.780331:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774738.780337:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774738.780344:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780349:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780353:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780359:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774738.780364:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774738.780370:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780374:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774738.780378:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774738.780382:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774738.780385:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780388:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774738.780390:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774738.780394:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780397:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774738.780400:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775774738.780404:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774738.780407:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774738.780407:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780411:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774738.780414:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774738.780416:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774738.780420:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774738.780426:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774738.780434:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099538688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774738.780458:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774738.780462:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774738.780470:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099538688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774738.780484:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774738.780486:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774738.780489:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774738.780491:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774738.780495:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774738.780499:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774738.780506:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16454 00000800:00000200:2.1:1775774738.780507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774738.780512:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000400:00000200:1.0:1775774738.780517:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774738.780556:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774738.780561:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774738.780566:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780570:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780575:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780580:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774738.780585:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774738.780586:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774738.780589:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774738.780591:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774738.780594:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774738.780599:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774738.780607:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780611:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780614:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780618:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774738.780624:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774738.780628:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780633:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774738.780636:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774738.780641:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774738.780647:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780651:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774738.780654:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774738.780658:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780661:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774738.780667:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774738.780673:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774738.780680:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774738.780683:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780685:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774738.780686:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774738.780688:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774738.780695:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774738.780701:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774738.780710:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099538816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774738.780728:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774738.780732:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774738.780737:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099538816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774738.780748:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774738.780753:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774738.780756:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774738.780759:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774738.780760:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774738.780764:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774738.780770:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16454 00000400:00000200:1.0:1775774738.780809:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774738.780813:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774738.780817:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774738.780823:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780827:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780832:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780835:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774738.780839:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774738.780842:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774738.780846:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774738.780849:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774738.780853:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774738.780857:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774738.780864:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780867:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774738.780870:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780873:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774738.780879:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774738.780885:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774738.780889:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774738.780893:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774738.780898:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774738.780904:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780909:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774738.780914:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774738.780921:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780927:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774738.780935:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774738.780941:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774738.780948:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774738.780952:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774738.780957:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774738.780960:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774738.780964:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774738.780970:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774738.780976:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774738.780986:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099538944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774738.780998:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774738.781003:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774738.781008:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099538944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774738.781018:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774738.781025:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16464) 00000800:00000200:2.1:1775774738.781119:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774738.781120:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775774738.781357:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774738.781358:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775774738.781554:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774738.781556:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775774738.781740:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774738.781743:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775774739.547975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774739.547979:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774739.547981:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774740.059892:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311127797, at 4311127808 00000100:00000010:1.0:1775774740.059906:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775774740.571970:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774740.571974:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774740.571978:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774741.596051:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774741.596055:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774741.596057:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774742.619971:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774742.619974:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774742.619976:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774743.643978:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774743.643982:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774743.643987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774743.900421:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774743.900425:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774743.901008:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774743.901010:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774743.901225:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774743.901227:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774743.901485:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774743.901486:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774743.901677:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774743.901679:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775774744.668009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774744.668012:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774744.668015:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774745.179878:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311132917, at 4311132928 00000100:00000010:1.0:1775774745.179893:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775774745.691923:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774745.691927:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774745.691930:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774746.715945:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774746.715949:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774746.715952:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774747.740051:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774747.740055:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774747.740059:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774748.763965:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774748.763969:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774748.763972:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774749.020458:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774749.020463:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774749.020950:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774749.020953:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774749.021140:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774749.021142:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774749.021343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774749.021358:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774749.021556:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774749.021558:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775774749.787993:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774749.787996:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774749.788000:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774750.299852:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311138037, at 4311138048 00000100:00000010:1.0:1775774750.299865:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775774750.811983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774750.811986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774750.811989:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774751.835958:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774751.835964:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774751.835968:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774752.860013:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774752.860016:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774752.860019:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774753.884089:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774753.884094:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774753.884098:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774754.140593:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774754.140603:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774754.141449:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774754.141450:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774754.141665:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774754.141666:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774754.141944:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774754.141947:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774754.142179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774754.142181:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775774754.908070:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774754.908076:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774754.908082:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774755.419919:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311143158, at 4311143168 00000100:00000010:1.0:1775774755.419933:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775774755.932054:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774755.932060:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774755.932063:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774756.955975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774756.955980:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774756.955984:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774757.980015:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774757.980019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774757.980023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774759.003993:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774759.003997:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774759.004001:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774759.260536:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774759.260559:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774759.261280:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774759.261283:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774759.261702:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774759.261705:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774759.261954:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774759.261956:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774759.262344:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774759.262347:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775774760.027979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774760.027983:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774760.027986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774760.539891:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311148277, at 4311148288 00000100:00000010:1.0:1775774760.539907:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775774761.052074:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774761.052080:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774761.052084:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774762.075913:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774762.075916:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774762.075919:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774763.099997:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774763.100001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774763.100003:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774764.124007:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774764.124010:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774764.124013:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774764.380392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774764.380398:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774764.380985:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774764.380987:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774764.381191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774764.381192:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774764.381367:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774764.381368:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774764.381535:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774764.381537:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775774765.147984:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774765.147989:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774765.147994:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774765.659940:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311153397, at 4311153408 00000100:00000010:1.0:1775774765.659965:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774766.171955:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774766.171958:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774766.171961:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774767.195995:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774767.195998:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774767.196001:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774768.220039:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774768.220060:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774768.220064:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774769.243909:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774769.243913:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774769.243916:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774769.499949:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774769.499960:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774769.499963:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774769.499967:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774769.499972:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774769.499977:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16484 00000400:00000200:1.0:1775774769.499986:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774769.500013:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774769.500016:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774769.500018:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500021:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.500023:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500025:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774769.500028:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774769.500029:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774769.500032:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774769.500034:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774769.500035:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774769.500037:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774769.500039:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500041:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.500042:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500044:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774769.500046:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774769.500048:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500049:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774769.500051:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774769.500054:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774769.500056:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.500057:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774769.500059:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774769.500067:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774769.500069:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774769.500073:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774769.500075:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.500078:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774769.500080:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774769.500085:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774769.500093:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774769.500101:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774769.500110:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099542272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774769.500138:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774769.500145:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774769.500153:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099542272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774769.500192:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774769.500197:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774769.500200:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774769.500202:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774769.500204:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774769.500207:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774769.500211:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16484 00000400:00000200:1.0:1775774769.500217:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774769.500221:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774769.500224:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774769.500228:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500231:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.500234:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500236:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774769.500238:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774769.500240:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774769.500246:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774769.500249:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774769.500254:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774769.500259:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774769.500268:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500272:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.500277:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500284:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774769.500291:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774769.500298:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500305:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774769.500309:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774769.500317:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774769.500323:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.500329:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774769.500335:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774769.500358:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.500360:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774769.500367:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774769.500371:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774769.500377:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774769.500379:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.500381:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774769.500382:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774769.500384:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774769.500390:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774769.500394:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774769.500402:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099542400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774769.500408:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774769.500411:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774769.500414:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099542400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774769.500431:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774769.500435:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774769.500442:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774769.500445:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774769.500447:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774769.500451:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774769.500455:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16484 00000400:00000200:1.0:1775774769.500461:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774769.500466:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774769.500470:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774769.500478:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500482:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.500487:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500491:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774769.500497:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774769.500499:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774769.500503:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774769.500507:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774769.500512:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774769.500519:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774769.500528:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500533:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.500535:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500537:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774769.500542:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774769.500549:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500553:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774769.500555:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774769.500559:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774769.500562:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.500566:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774769.500568:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774769.500571:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.500573:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774769.500579:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774769.500582:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774769.500586:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774769.500588:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.500590:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774769.500593:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.1:1775774769.500594:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775774769.500597:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774769.500600:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 02000000:00000010:1.0:1775774769.500603:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774769.500609:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774769.500619:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099542528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774769.500636:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774769.500640:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774769.500668:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099542528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774769.500689:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774769.500698:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774769.500707:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774769.500710:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774769.500712:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774769.500731:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774769.500738:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16484 00000400:00000200:1.0:1775774769.500747:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774769.500756:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774769.500762:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774769.500773:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500860:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.500868:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500877:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774769.500881:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774769.500887:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774769.500890:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774769.500897:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774769.500904:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774769.500910:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774769.500924:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500940:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.500950:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.500959:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774769.500966:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774769.500977:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774769.501005:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774769.501027:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774769.501033:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774769.501039:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.501043:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774769.501047:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774769.501051:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.501053:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774769.501056:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774769.501061:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774769.501065:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774769.501067:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.501071:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774769.501073:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774769.501075:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774769.501081:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774769.501088:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774769.501099:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099542656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774769.501118:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774769.501123:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774769.501133:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099542656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774769.501152:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774769.501156:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774769.501161:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774769.501164:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774769.501166:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774769.501169:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774769.501175:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16484 00000400:00000200:1.0:1775774769.501183:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774769.501188:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774769.501191:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774769.501197:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.501201:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.501206:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.501212:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774769.501216:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774769.501219:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774769.501222:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774769.501234:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774769.501236:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774769.501238:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774769.501243:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.501246:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774769.501250:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774769.501255:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774769.501258:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774769.501261:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774769.501264:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774769.501267:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774769.501271:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774769.501274:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.501276:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774769.501279:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774769.501282:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.501283:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774769.501286:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774769.501289:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774769.501294:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774769.501297:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774769.501299:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774769.501302:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774769.501306:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774769.501311:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774769.501316:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774769.501323:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099542784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774769.501350:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774769.501353:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774769.501358:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099542784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774769.501373:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774769.501388:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16494) 00000800:00000200:2.1:1775774769.501456:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774769.501458:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774769.501758:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774769.501762:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774769.502114:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774769.502115:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774769.502371:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774769.502373:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775774770.267970:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774770.267974:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774770.267991:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774770.779855:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311158518, at 4311158528 00000100:00000010:1.0:1775774770.779868:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774771.292020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774771.292024:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774771.292027:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774772.316056:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774772.316059:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774772.316062:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774773.340131:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774773.340135:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774773.340138:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774774.363955:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774774.363958:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774774.363961:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774774.620478:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774774.620482:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775774774.621111:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774774.621113:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775774774.621296:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774774.621297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775774774.621490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774774.621492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775774774.621728:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774774.621729:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000020:00000001:1.0:1775774775.388000:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774775.388002:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774775.388005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774775.899941:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311163637, at 4311163648 00000100:00000010:1.0:1775774775.899957:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775774776.411974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774776.411978:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774776.411982:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774777.436034:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774777.436038:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774777.436041:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774778.460007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774778.460011:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774778.460014:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774779.484019:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774779.484024:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774779.484028:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774779.739949:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774779.739960:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774779.739963:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774779.739968:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774779.739974:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774779.739983:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16495 00000400:00000200:1.0:1775774779.739995:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774779.740003:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774779.740008:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774779.740012:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740017:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740020:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740023:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774779.740028:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774779.740030:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774779.740034:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774779.740038:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774779.740041:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774779.740044:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774779.740048:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740050:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740052:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740054:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774779.740057:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774779.740061:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740063:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774779.740066:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774779.740069:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774779.740072:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740084:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774779.740086:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774779.740101:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774779.740105:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774779.740108:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774779.740110:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740112:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774779.740116:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774779.740120:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774779.740129:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774779.740136:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774779.740145:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099543552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774779.740154:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774779.740159:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774779.740163:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099543552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774779.740183:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774779.740187:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774779.740190:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774779.740194:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774779.740195:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774779.740198:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774779.740200:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16495 00000400:00000200:1.0:1775774779.740203:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774779.740206:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774779.740207:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774779.740210:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740212:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740213:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740215:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774779.740216:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774779.740217:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774779.740218:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774779.740220:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774779.740221:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774779.740223:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774779.740225:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740226:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740227:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740228:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774779.740229:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774779.740231:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740232:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774779.740233:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774779.740239:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774779.740241:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740242:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774779.740247:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774779.740249:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740250:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774779.740253:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774779.740255:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774779.740257:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774779.740258:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740259:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774779.740260:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774779.740262:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774779.740265:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775774779.740269:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774779.740275:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099543680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774779.740284:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774779.740288:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774779.740295:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099543680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774779.740307:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774779.740310:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774779.740316:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774779.740318:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774779.740320:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774779.740323:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774779.740328:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16495 00000400:00000200:1.0:1775774779.740333:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774779.740339:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774779.740341:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774779.740345:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740348:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740353:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740357:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774779.740358:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774779.740359:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774779.740361:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774779.740363:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774779.740366:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774779.740369:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774779.740371:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740375:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740379:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740381:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774779.740384:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774779.740386:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740388:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774779.740389:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774779.740393:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774779.740395:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740396:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774779.740398:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774779.740400:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740401:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774779.740404:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774779.740406:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774779.740407:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774779.740408:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740409:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774779.740409:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774779.740411:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774779.740415:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775774779.740417:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774779.740420:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099543808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774779.740425:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774779.740426:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774779.740429:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099543808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774779.740433:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774779.740435:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774779.740438:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774779.740438:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774779.740440:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774779.740442:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774779.740443:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16495 00000400:00000200:1.0:1775774779.740446:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774779.740448:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774779.740449:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774779.740451:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740452:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740453:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740454:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774779.740456:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774779.740456:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774779.740457:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774779.740458:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774779.740460:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774779.740461:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774779.740463:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740464:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740465:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740466:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774779.740471:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774779.740475:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740479:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774779.740481:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774779.740484:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774779.740489:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740493:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774779.740497:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774779.740500:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740503:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774779.740509:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774779.740512:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774779.740516:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774779.740517:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740519:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774779.740521:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774779.740523:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774779.740527:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774779.740533:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774779.740537:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099543936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774779.740546:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774779.740550:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774779.740555:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099543936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774779.740567:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774779.740572:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774779.740579:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774779.740584:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774779.740587:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774779.740593:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774779.740601:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16495 00000400:00000200:1.0:1775774779.740611:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774779.740615:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774779.740619:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774779.740622:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740625:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740629:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740635:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774779.740639:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774779.740640:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774779.740642:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774779.740643:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774779.740647:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774779.740651:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774779.740655:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740658:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774779.740662:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740665:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774779.740667:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774779.740670:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774779.740672:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774779.740675:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774779.740678:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774779.740679:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740681:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774779.740684:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774779.740687:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740691:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774779.740695:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000800:00000200:2.1:1775774779.740696:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:1.0:1775774779.740698:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774779.740703:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774779.740704:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774779.740706:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774779.740707:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774779.740709:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774779.740712:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774779.740715:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774779.740719:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099544064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774779.740725:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774779.740728:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774779.740731:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099544064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774779.740743:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774779.740749:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16505) 00000800:00000200:2.1:1775774779.740955:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774779.741074:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774779.741234:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774779.741446:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775774780.507965:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774780.507969:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774780.507973:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774781.019885:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311168757, at 4311168768 00000100:00000010:1.0:1775774781.019902:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775774781.532033:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774781.532038:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774781.532042:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774782.556058:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774782.556063:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774782.556067:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774783.579966:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774783.579970:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774783.579974:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774784.603967:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774784.603971:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774784.603975:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774784.861761:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774784.861764:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774784.862144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774784.862146:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774784.862359:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774784.862361:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774784.862552:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774784.862553:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774784.862735:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774784.862738:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775774785.627986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774785.627990:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774785.627994:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774786.139867:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311173879, at 4311173888 00000100:00000010:1.0:1775774786.139880:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775774786.651985:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774786.651990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774786.651994:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774787.676020:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774787.676023:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774787.676026:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774788.700006:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774788.700009:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774788.700012:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774789.723987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774789.723990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774789.723993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774789.980571:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774789.980578:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774789.981234:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774789.981236:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774789.981555:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774789.981558:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774789.981965:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774789.981979:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774789.982300:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774789.982303:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775774790.747989:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774790.747992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774790.747994:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774791.259905:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311178998, at 4311179008 00000100:00000010:1.0:1775774791.259929:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775774791.771963:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774791.771966:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774791.771969:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774792.796070:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774792.796075:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774792.796079:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774793.819989:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774793.819995:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774793.820001:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774794.844023:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774794.844027:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774794.844032:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774795.100320:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774795.100327:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774795.100961:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774795.100963:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774795.101147:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774795.101149:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774795.101353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774795.101356:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774795.101659:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774795.101661:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775774795.867935:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774795.867939:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774795.867942:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774796.379882:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311184117, at 4311184128 00000100:00000010:1.0:1775774796.379895:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775774796.891978:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774796.891982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774796.891986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774797.915949:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774797.915952:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774797.915954:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774798.939965:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774798.939968:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774798.939970:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774799.963951:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774799.963955:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774799.963959:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774800.219910:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774800.219920:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774800.219923:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774800.219927:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774800.219932:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774800.219939:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16515 00000400:00000200:1.0:1775774800.219950:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774800.219957:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774800.219960:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774800.219964:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.219967:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.219970:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.219973:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774800.219977:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774800.219979:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774800.219982:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774800.219985:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774800.219988:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774800.219990:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774800.219993:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.219996:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.219998:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220000:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774800.220003:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774800.220006:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220010:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774800.220013:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774800.220017:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774800.220020:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220023:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774800.220025:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774800.220035:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774800.220039:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774800.220043:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774800.220045:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220048:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774800.220051:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774800.220056:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774800.220064:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774800.220070:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774800.220079:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099546112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774800.220090:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774800.220094:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774800.220099:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099546112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774800.220125:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774800.220129:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774800.220132:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774800.220134:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774800.220136:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774800.220139:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774800.220141:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16515 00000400:00000200:1.0:1775774800.220146:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774800.220151:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774800.220154:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774800.220158:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220160:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.220164:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220169:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774800.220172:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774800.220174:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774800.220176:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774800.220178:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774800.220181:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774800.220184:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774800.220193:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220195:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.220200:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220204:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774800.220210:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774800.220217:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220223:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774800.220228:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774800.220235:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774800.220242:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220247:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774800.220253:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774800.220258:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220261:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774800.220269:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774800.220274:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774800.220279:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774800.220282:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220285:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774800.220288:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774800.220291:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774800.220296:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774800.220302:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774800.220312:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099546240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774800.220343:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774800.220349:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774800.220358:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099546240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774800.220376:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774800.220381:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774800.220388:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774800.220393:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774800.220397:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774800.220405:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774800.220411:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16515 00000400:00000200:1.0:1775774800.220420:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774800.220424:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774800.220427:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774800.220431:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220435:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.220442:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220447:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774800.220466:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774800.220470:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774800.220473:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774800.220476:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774800.220482:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774800.220485:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774800.220493:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220500:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775774800.220505:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775774800.220507:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775774800.220509:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000001:1.0:1775774800.220513:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774800.220521:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774800.220529:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220533:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774800.220538:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774800.220544:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774800.220550:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220553:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774800.220556:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774800.220558:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220561:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774800.220567:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774800.220573:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774800.220579:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774800.220581:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220586:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774800.220590:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774800.220594:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774800.220600:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774800.220605:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774800.220616:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099546368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774800.220628:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774800.220632:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774800.220639:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099546368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774800.220661:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774800.220666:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774800.220677:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774800.220681:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774800.220684:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774800.220688:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774800.220693:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16515 00000400:00000200:1.0:1775774800.220703:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774800.220711:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774800.220717:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774800.220727:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220732:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.220739:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220744:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774800.220749:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774800.220753:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774800.220758:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774800.220761:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774800.220767:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774800.220772:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774800.220829:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220836:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.220842:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220847:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774800.220851:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774800.220855:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774800.220861:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774800.220866:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774800.220870:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774800.220875:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220880:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774800.220884:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774800.220890:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220893:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774800.220899:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774800.220904:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774800.220910:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774800.220914:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.220918:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774800.220921:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774800.220925:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774800.220932:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774800.220937:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774800.220947:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099546496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774800.220962:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774800.220967:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774800.220976:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099546496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774800.220999:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774800.221004:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774800.221010:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774800.221013:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774800.221016:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774800.221021:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774800.221027:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16515 00000400:00000200:1.0:1775774800.221037:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774800.221044:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774800.221049:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774800.221056:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.221061:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.221067:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.221073:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774800.221078:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774800.221081:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774800.221086:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774800.221090:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774800.221095:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774800.221102:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774800.221110:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.221117:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774800.221125:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774800.221132:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774800.221137:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774800.221145:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774800.221152:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774800.221156:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774800.221161:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774800.221167:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.221169:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774800.221173:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774800.221180:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.221183:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774800.221191:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000800:00000200:2.1:1775774800.221192:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:1.0:1775774800.221197:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000800:00000200:2.1:1775774800.221198:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 02000000:00000001:1.0:1775774800.221203:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774800.221207:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774800.221211:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774800.221215:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774800.221218:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774800.221224:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774800.221231:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774800.221242:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099546624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774800.221265:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774800.221270:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774800.221279:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099546624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774800.221299:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774800.221305:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16525) 00000800:00000200:2.1:1775774800.221516:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774800.221517:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775774800.221749:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774800.221751:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775774800.221999:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774800.222002:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775774800.987977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774800.987981:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774800.987983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774801.499858:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311189237, at 4311189248 00000100:00000010:1.0:1775774801.499874:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775774802.012003:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774802.012009:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774802.012013:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774803.035974:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774803.035978:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774803.035981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774804.059972:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774804.059978:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774804.059983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774805.084031:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774805.084034:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774805.084038:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774805.339946:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774805.339956:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774805.339959:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774805.339964:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774805.339968:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774805.339983:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16520 00000400:00000200:1.0:1775774805.339997:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774805.340003:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774805.340007:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774805.340010:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340013:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340016:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340017:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774805.340021:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774805.340023:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774805.340026:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774805.340029:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774805.340031:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774805.340033:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774805.340035:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340037:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340038:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340039:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774805.340041:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774805.340044:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340045:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774805.340048:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774805.340051:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774805.340053:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340055:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774805.340056:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774805.340067:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774805.340070:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774805.340074:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774805.340075:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340078:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774805.340080:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774805.340085:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774805.340093:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775774805.340100:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774805.340109:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099546752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774805.340119:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774805.340123:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774805.340127:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099546752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774805.340148:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774805.340151:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774805.340154:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774805.340155:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774805.340156:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774805.340159:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774805.340160:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16520 00000400:00000200:1.0:1775774805.340163:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774805.340165:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774805.340167:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774805.340168:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340170:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340171:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340173:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774805.340174:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774805.340175:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774805.340176:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774805.340178:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774805.340179:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774805.340180:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774805.340183:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340185:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340187:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340189:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774805.340191:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774805.340193:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340194:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774805.340195:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774805.340201:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774805.340209:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340216:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774805.340222:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774805.340228:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340232:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774805.340237:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774805.340240:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774805.340243:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774805.340245:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340248:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774805.340249:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774805.340251:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774805.340256:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775774805.340261:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774805.340270:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099546880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774805.340289:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774805.340292:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774805.340297:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099546880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774805.340308:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774805.340312:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774805.340314:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774805.340316:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774805.340317:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774805.340322:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774805.340326:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16520 00000400:00000200:1.0:1775774805.340329:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774805.340333:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774805.340337:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774805.340339:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340340:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340345:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340346:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774805.340348:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774805.340349:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774805.340351:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774805.340353:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774805.340355:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774805.340357:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774805.340359:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340360:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340361:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340382:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774805.340385:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774805.340389:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340392:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774805.340393:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774805.340396:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774805.340398:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340401:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774805.340403:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774805.340408:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340412:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774805.340417:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774805.340420:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774805.340424:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774805.340425:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340428:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774805.340429:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774805.340431:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774805.340439:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774805.340443:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774805.340448:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099547008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774805.340461:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774805.340463:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774805.340469:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099547008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774805.340478:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774805.340480:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774805.340483:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774805.340486:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774805.340488:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774805.340489:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774805.340492:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16520 00000400:00000200:1.0:1775774805.340498:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774805.340504:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774805.340507:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774805.340512:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340516:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340520:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340523:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774805.340527:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774805.340528:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774805.340531:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774805.340534:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774805.340536:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774805.340541:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774805.340547:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340548:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340550:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340552:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774805.340556:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774805.340562:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340565:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774805.340570:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774805.340572:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774805.340575:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340576:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774805.340579:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774805.340585:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340586:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774805.340589:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774805.340595:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774805.340597:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774805.340599:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340603:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774805.340606:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774805.340610:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774805.340615:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000800:00000200:2.1:1775774805.340621:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775774805.340622:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774805.340627:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000100:00100000:1.0:1775774805.340632:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099547136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774805.340650:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774805.340653:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774805.340661:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099547136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774805.340674:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774805.340679:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774805.340683:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774805.340685:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774805.340689:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774805.340692:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774805.340696:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16520 00000400:00000200:1.0:1775774805.340703:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774805.340708:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774805.340713:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774805.340718:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340721:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340723:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340726:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774805.340729:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774805.340730:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774805.340731:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774805.340734:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774805.340738:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774805.340742:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774805.340748:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340753:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774805.340755:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340761:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774805.340765:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774805.340769:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774805.340771:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774805.340776:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774805.340804:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774805.340807:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340810:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774805.340812:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774805.340815:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340817:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774805.340825:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774805.340830:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774805.340835:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774805.340839:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774805.340842:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774805.340845:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774805.340847:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774805.340855:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774805.340859:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774805.340869:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099547264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774805.340880:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774805.340884:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774805.340889:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099547264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774805.340897:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774805.340902:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16530) 00000800:00000200:2.1:1775774805.341212:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774805.341215:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775774805.341523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774805.341526:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775774805.341825:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774805.341828:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775774805.342100:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774805.342103:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775774806.107984:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774806.107988:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774806.107991:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774806.619878:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311194358, at 4311194368 00000100:00000010:1.0:1775774806.619895:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775774807.131986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774807.131989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774807.131993:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774808.155985:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774808.155991:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774808.155995:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774809.179992:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774809.179997:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774809.180000:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774810.203999:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774810.204003:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774810.204006:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774810.459898:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774810.459906:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774810.459909:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774810.459913:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774810.459916:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774810.459922:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16525 00000400:00000200:1.0:1775774810.459931:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774810.459936:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774810.459938:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774810.459940:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.459943:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.459945:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.459946:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774810.459950:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774810.459951:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774810.459953:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774810.459955:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774810.459956:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774810.459958:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774810.459960:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.459961:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.459963:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.459964:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774810.459966:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774810.459969:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774810.459971:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774810.459973:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774810.459975:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774810.459977:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.459979:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774810.459980:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774810.459990:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774810.459992:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774810.459996:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774810.459997:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.459999:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774810.460001:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774810.460005:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774810.460011:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774810.460017:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774810.460024:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099547392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774810.460032:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774810.460036:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774810.460039:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099547392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774810.460058:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774810.460060:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774810.460063:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774810.460064:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774810.460065:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774810.460067:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774810.460069:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16525 00000400:00000200:1.0:1775774810.460071:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774810.460072:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774810.460074:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774810.460078:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460080:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.460083:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460086:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774810.460088:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774810.460090:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774810.460092:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774810.460094:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774810.460097:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774810.460099:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774810.460104:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460109:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.460113:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460117:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774810.460121:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774810.460129:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460136:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774810.460140:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774810.460146:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774810.460149:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460152:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774810.460158:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774810.460162:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460166:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774810.460169:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774810.460173:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774810.460178:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774810.460181:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460182:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774810.460183:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774810.460185:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774810.460190:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774810.460197:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774810.460203:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099547520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774810.460213:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774810.460215:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774810.460221:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099547520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774810.460251:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774810.460255:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774810.460261:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774810.460263:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774810.460265:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774810.460268:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774810.460271:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16525 00000400:00000200:1.0:1775774810.460293:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774810.460297:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774810.460299:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774810.460302:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460304:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.460305:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460308:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774810.460311:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774810.460313:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774810.460314:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774810.460317:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774810.460321:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774810.460323:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774810.460328:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460330:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.460334:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460336:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774810.460339:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774810.460345:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460349:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774810.460353:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774810.460357:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774810.460362:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460364:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774810.460366:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774810.460369:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460373:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774810.460378:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775774810.460383:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774810.460387:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774810.460388:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460389:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774810.460390:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774810.460394:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774810.460397:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774810.460402:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774810.460408:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099547648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775774810.460413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774810.460419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000100:00000001:1.0:1775774810.460423:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774810.460427:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774810.460433:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099547648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774810.460443:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774810.460448:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774810.460452:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774810.460453:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774810.460456:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774810.460461:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774810.460465:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16525 00000400:00000200:1.0:1775774810.460474:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774810.460483:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774810.460486:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774810.460494:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460500:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.460505:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460510:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774810.460515:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774810.460518:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774810.460522:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774810.460527:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774810.460529:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774810.460532:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774810.460535:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460537:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.460540:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460542:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774810.460544:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774810.460548:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460551:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774810.460554:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774810.460558:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774810.460563:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460566:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774810.460569:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774810.460573:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460575:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774810.460581:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774810.460586:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774810.460588:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774810.460591:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460593:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774810.460595:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774810.460598:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774810.460602:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774810.460608:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774810.460617:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099547776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774810.460629:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774810.460632:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774810.460639:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099547776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774810.460651:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774810.460654:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774810.460662:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774810.460665:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774810.460668:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774810.460670:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774810.460675:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16525 00000400:00000200:1.0:1775774810.460683:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774810.460687:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774810.460691:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774810.460696:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460699:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.460701:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460704:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774810.460707:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774810.460708:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774810.460709:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774810.460710:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774810.460711:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774810.460712:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774810.460714:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460715:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774810.460719:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460721:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774810.460724:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774810.460728:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774810.460730:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774810.460733:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774810.460735:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774810.460737:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460740:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774810.460743:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774810.460745:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460749:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774810.460752:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774810.460754:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774810.460758:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774810.460760:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774810.460762:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774810.460764:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774810.460767:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774810.460772:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774810.460777:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774810.460811:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099547904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774810.460833:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774810.460836:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774810.460842:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099547904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774810.460854:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774810.460861:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16535) 00000800:00000200:2.1:1775774810.460988:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774810.460990:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775774810.461202:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774810.461204:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775774810.461401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774810.461403:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775774810.461612:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774810.461614:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000020:00000001:1.0:1775774811.227984:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774811.227988:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774811.227991:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774811.740017:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311199477, at 4311199488 00000100:00000010:1.0:1775774811.740029:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775774812.252014:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774812.252020:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774812.252025:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774813.276020:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774813.276024:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774813.276026:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774814.299989:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774814.299992:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774814.299995:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774815.323939:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774815.323942:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774815.323951:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774815.580424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774815.580428:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774815.581087:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774815.581089:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774815.581304:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774815.581306:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774815.581479:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774815.581480:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774815.581754:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774815.581756:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775774816.348022:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774816.348026:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774816.348030:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774816.859907:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311204597, at 4311204608 00000100:00000010:1.0:1775774816.859923:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775774817.372041:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774817.372046:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774817.372050:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774818.395981:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774818.395986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774818.395990:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774819.420091:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774819.420096:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774819.420100:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774820.444014:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774820.444018:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774820.444021:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774820.699935:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774820.699953:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774820.699955:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774820.699959:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774820.699963:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774820.699968:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16536 00000400:00000200:1.0:1775774820.699977:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774820.699982:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774820.699985:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774820.699987:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.699989:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.699991:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.699993:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774820.699996:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774820.699997:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774820.700000:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774820.700002:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774820.700003:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774820.700005:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774820.700006:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700008:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700009:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700010:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774820.700013:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774820.700015:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700017:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774820.700019:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774820.700022:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774820.700024:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700025:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774820.700027:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774820.700035:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774820.700038:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774820.700040:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774820.700042:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700045:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774820.700047:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774820.700050:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774820.700059:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775774820.700064:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774820.700071:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099548672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774820.700079:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774820.700083:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774820.700086:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099548672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774820.700115:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774820.700118:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774820.700121:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774820.700122:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774820.700123:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774820.700125:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774820.700127:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16536 00000400:00000200:1.0:1775774820.700130:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774820.700133:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774820.700134:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774820.700137:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700138:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700140:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700142:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774820.700143:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774820.700144:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774820.700146:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774820.700147:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774820.700149:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774820.700151:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774820.700153:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700155:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700156:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700158:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774820.700160:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774820.700161:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700163:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774820.700165:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774820.700169:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774820.700172:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700175:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774820.700185:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774820.700190:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700194:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774820.700198:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774820.700200:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774820.700201:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774820.700202:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700203:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774820.700205:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774820.700208:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774820.700213:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775774820.700217:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774820.700223:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099548800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774820.700232:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774820.700236:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774820.700241:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099548800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774820.700253:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774820.700259:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774820.700264:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774820.700267:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774820.700270:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774820.700275:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774820.700280:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16536 00000400:00000200:1.0:1775774820.700288:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774820.700291:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774820.700294:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774820.700299:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700302:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700307:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700311:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774820.700315:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774820.700316:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774820.700318:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774820.700319:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774820.700321:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774820.700324:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774820.700328:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700329:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700331:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700333:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774820.700337:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774820.700340:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700342:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774820.700343:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774820.700351:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774820.700354:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700357:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774820.700360:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774820.700363:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700365:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774820.700368:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775774820.700372:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774820.700376:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774820.700378:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700382:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774820.700385:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774820.700388:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774820.700396:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774820.700402:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774820.700411:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099548928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774820.700425:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774820.700430:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774820.700437:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099548928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774820.700450:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774820.700453:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774820.700457:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774820.700459:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774820.700461:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774820.700464:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774820.700470:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16536 00000400:00000200:1.0:1775774820.700475:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774820.700482:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774820.700486:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774820.700490:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700494:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700498:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700501:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774820.700502:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774820.700503:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774820.700505:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774820.700508:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774820.700513:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774820.700518:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774820.700524:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700529:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700534:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700537:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774820.700542:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774820.700549:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700554:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774820.700558:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774820.700562:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774820.700566:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700568:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.1:1775774820.700568:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00100000:1.0:1775774820.700571:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774820.700574:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774820.700575:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1775774820.700576:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774820.700580:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774820.700583:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774820.700586:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774820.700588:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700590:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774820.700592:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774820.700595:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774820.700599:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774820.700604:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774820.700611:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099549056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774820.700626:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774820.700629:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774820.700632:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099549056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774820.700642:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774820.700647:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774820.700652:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774820.700655:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774820.700657:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774820.700661:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774820.700665:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16536 00000400:00000200:1.0:1775774820.700672:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774820.700674:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774820.700676:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774820.700677:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700679:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700680:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700683:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774820.700685:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774820.700687:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774820.700689:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774820.700692:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774820.700696:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774820.700699:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774820.700706:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700708:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774820.700711:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700713:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774820.700717:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774820.700719:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774820.700721:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774820.700722:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774820.700725:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774820.700727:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700729:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774820.700731:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774820.700734:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700736:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774820.700739:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774820.700744:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774820.700745:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774820.700746:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774820.700747:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774820.700749:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774820.700751:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774820.700756:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774820.700758:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774820.700761:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099549184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774820.700768:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774820.700770:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774820.700774:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099549184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774820.700833:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774820.700841:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16546) 00000800:00000200:2.1:1775774820.701198:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774820.701201:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774820.701524:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774820.701527:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774820.701866:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774820.701869:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774820.702180:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774820.702183:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775774821.467969:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774821.467973:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774821.467976:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774821.979875:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311209717, at 4311209728 00000100:00000010:1.0:1775774821.979887:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775774822.491972:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774822.491975:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774822.491978:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774823.516008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774823.516013:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774823.516017:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774824.540009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774824.540013:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774824.540016:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774825.563934:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774825.563938:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774825.563941:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774825.820497:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774825.820503:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774825.821310:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774825.821313:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774825.821613:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774825.821616:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774825.821951:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774825.821953:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774825.822311:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774825.822314:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775774826.587956:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774826.587959:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774826.587962:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774827.099852:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311214837, at 4311214848 00000100:00000010:1.0:1775774827.099865:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774827.611956:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774827.611959:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774827.611962:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774828.635997:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774828.636002:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774828.636005:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774829.660065:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774829.660070:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774829.660074:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774830.684009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774830.684012:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774830.684015:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774830.940585:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774830.940594:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774830.941317:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774830.941318:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774830.941510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774830.941513:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774830.941727:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774830.941729:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774830.941956:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774830.941958:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775774831.708037:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774831.708041:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774831.708044:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774832.219887:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311219958, at 4311219968 00000100:00000010:1.0:1775774832.219923:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774832.732035:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774832.732041:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774832.732045:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774833.756059:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774833.756062:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774833.756065:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774834.780005:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774834.780008:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774834.780010:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774835.804012:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774835.804016:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774835.804019:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774836.060531:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774836.060537:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774836.061188:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774836.061190:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774836.061385:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774836.061387:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774836.061733:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774836.061736:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774836.062084:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774836.062087:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775774836.827988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774836.827991:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774836.827994:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774837.339897:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311225077, at 4311225088 00000100:00000010:1.0:1775774837.339919:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775774838.875993:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774838.875998:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774838.876002:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774839.900056:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774839.900061:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774839.900065:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774841.180671:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774841.180681:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775774841.181459:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774841.181462:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775774841.181813:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774841.181816:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775774841.182115:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774841.182118:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775774841.182424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774841.182427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775774841.947996:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774841.947999:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774841.948002:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774842.459906:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311230198, at 4311230208 00000100:00000010:1.0:1775774842.459920:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775774842.972084:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774842.972089:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774842.972093:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774845.019990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774845.019993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774845.019996:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774846.044006:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774846.044010:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774846.044014:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774846.299926:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774846.299936:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774846.299939:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774846.299944:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774846.299948:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774846.299956:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16561 00000400:00000200:1.0:1775774846.299967:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774846.299974:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774846.299977:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774846.299980:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.299983:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.299985:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.299987:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774846.299991:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774846.299992:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774846.299995:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774846.299998:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774846.299999:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774846.300001:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774846.300002:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300004:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300005:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300007:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774846.300009:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774846.300012:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300014:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774846.300016:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774846.300019:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774846.300022:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300025:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774846.300028:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774846.300043:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774846.300048:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774846.300052:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774846.300055:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300058:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774846.300062:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774846.300068:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774846.300077:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774846.300086:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774846.300096:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099551872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774846.300105:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774846.300110:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774846.300114:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099551872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774846.300136:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774846.300139:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774846.300142:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774846.300143:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774846.300144:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774846.300147:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774846.300149:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16561 00000400:00000200:1.0:1775774846.300152:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774846.300154:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774846.300156:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774846.300159:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300161:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300162:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300164:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774846.300167:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774846.300169:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774846.300170:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774846.300172:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774846.300173:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774846.300175:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774846.300177:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300178:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300179:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300180:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774846.300182:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774846.300185:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300189:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774846.300191:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774846.300197:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774846.300200:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300201:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774846.300206:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774846.300211:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300216:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774846.300222:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774846.300225:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774846.300229:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774846.300232:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300234:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774846.300235:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774846.300239:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774846.300245:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774846.300250:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774846.300259:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099552000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774846.300270:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774846.300274:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774846.300277:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099552000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774846.300289:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774846.300291:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774846.300296:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774846.300298:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774846.300299:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774846.300301:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774846.300305:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16561 00000400:00000200:1.0:1775774846.300307:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774846.300310:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774846.300314:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774846.300316:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300318:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300319:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300322:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774846.300325:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774846.300327:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774846.300331:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774846.300333:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774846.300335:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774846.300337:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774846.300344:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300348:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300352:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300356:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774846.300358:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774846.300365:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300371:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774846.300375:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774846.300379:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774846.300382:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300385:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774846.300386:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774846.300388:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300392:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774846.300396:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774846.300399:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774846.300401:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774846.300403:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300406:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774846.300408:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774846.300409:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774846.300413:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774846.300417:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774846.300422:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099552128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774846.300433:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774846.300435:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774846.300441:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099552128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774846.300455:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774846.300459:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774846.300465:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774846.300467:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774846.300468:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774846.300470:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774846.300475:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16561 00000400:00000200:1.0:1775774846.300490:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774846.300497:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774846.300502:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774846.300511:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:2.1:1775774846.300512:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775774846.300517:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300522:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300524:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774846.300527:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774846.300528:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774846.300530:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774846.300533:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774846.300538:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774846.300542:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774846.300548:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300551:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300552:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300557:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774846.300559:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774846.300562:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300564:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774846.300567:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774846.300570:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774846.300572:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300574:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774846.300576:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774846.300580:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300582:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774846.300588:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774846.300591:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774846.300593:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774846.300596:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300599:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774846.300600:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774846.300604:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774846.300612:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774846.300615:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774846.300619:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099552256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774846.300632:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774846.300636:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774846.300642:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099552256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774846.300678:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774846.300693:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774846.300698:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774846.300700:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774846.300703:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774846.300709:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774846.300715:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16561 00000400:00000200:1.0:1775774846.300722:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774846.300726:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774846.300728:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774846.300733:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300735:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300737:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300742:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774846.300746:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774846.300748:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774846.300750:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774846.300754:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774846.300756:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774846.300759:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774846.300764:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300767:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774846.300771:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300774:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774846.300777:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774846.300819:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774846.300821:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774846.300822:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774846.300827:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774846.300831:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300835:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774846.300838:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774846.300843:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300845:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774846.300850:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774846.300854:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774846.300858:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774846.300870:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774846.300872:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774846.300874:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774846.300876:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774846.300881:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775774846.300886:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774846.300894:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099552384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774846.300902:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774846.300906:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774846.300911:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099552384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774846.300933:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774846.300941:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16571) 00000800:00000200:2.1:1775774846.300974:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774846.301147:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774846.301323:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774846.301472:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775774847.579881:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311235317, at 4311235328 00000100:00000010:1.0:1775774847.579899:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774848.092030:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774848.092035:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774848.092038:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774849.115961:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774849.115964:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774849.115968:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774851.164001:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774851.164005:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774851.164009:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774851.420656:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774851.420661:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774851.421192:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774851.421195:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774851.421487:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774851.421490:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774851.422195:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774851.422198:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774851.422440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774851.422442:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775774852.187939:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774852.187941:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774852.187944:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774852.699895:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311240438, at 4311240448 00000100:00000010:1.0:1775774852.699911:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775774854.235995:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774854.236000:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774854.236004:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774855.259983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774855.259986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774855.259989:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774856.540427:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774856.540433:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774856.541132:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774856.541138:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774856.541399:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774856.541401:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774856.541626:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774856.541627:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774856.541916:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774856.541918:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775774857.308039:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774857.308044:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774857.308049:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774857.819902:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311245557, at 4311245568 00000100:00000010:1.0:1775774857.819916:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775774858.331986:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774858.331991:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774858.331995:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774860.379965:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774860.379968:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774860.379971:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774861.404017:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774861.404020:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774861.404023:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774861.660359:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774861.660364:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775774861.661039:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774861.661040:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775774861.661270:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774861.661271:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775774861.661471:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774861.661472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775774861.661772:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774861.661774:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000100:00000040:1.0:1775774862.939861:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311250677, at 4311250688 00000100:00000010:1.0:1775774862.939873:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775774863.451998:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774863.452002:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774863.452006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774864.475942:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774864.475945:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774864.475948:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774866.523958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774866.523961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774866.523964:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774866.780482:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774866.780485:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774866.781229:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774866.781234:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774866.781526:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774866.781529:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774866.781893:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774866.781895:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774866.782148:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774866.782151:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1775774867.547970:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774867.547983:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774867.547987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774868.059882:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311255797, at 4311255808 00000100:00000010:1.0:1775774868.059914:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775774869.595947:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774869.595951:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774869.595954:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774870.619951:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774870.619953:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774870.619956:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774871.900640:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774871.900649:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775774871.901273:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774871.901274:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775774871.901492:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774871.901494:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775774871.901737:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774871.901739:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775774871.902043:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774871.902045:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775774872.668020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774872.668023:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774872.668026:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774873.179869:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311260918, at 4311260928 00000100:00000010:1.0:1775774873.179885:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774873.691937:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774873.691941:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774873.691945:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774875.739966:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774875.739969:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774875.739971:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774876.764017:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774876.764021:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774876.764024:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774877.019956:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774877.019968:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774877.019971:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774877.019976:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774877.019980:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774877.019988:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16592 00000400:00000200:1.0:1775774877.019999:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774877.020005:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774877.020008:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774877.020011:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020014:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020016:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020018:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774877.020021:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774877.020023:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774877.020026:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774877.020029:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774877.020030:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774877.020031:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774877.020033:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020035:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020036:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020038:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774877.020040:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774877.020043:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020045:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774877.020047:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774877.020050:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774877.020053:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020055:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774877.020057:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774877.020069:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774877.020072:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774877.020076:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774877.020077:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020080:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774877.020082:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774877.020087:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774877.020096:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774877.020104:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774877.020135:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099555712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774877.020145:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774877.020149:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774877.020153:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099555712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774877.020178:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774877.020181:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774877.020183:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774877.020184:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774877.020186:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774877.020188:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774877.020190:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16592 00000400:00000200:1.0:1775774877.020193:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774877.020195:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774877.020197:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774877.020199:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020201:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020203:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020205:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774877.020207:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774877.020209:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774877.020210:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774877.020212:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774877.020213:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774877.020215:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774877.020217:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020218:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020220:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020221:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774877.020223:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774877.020225:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020228:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774877.020231:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774877.020243:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774877.020246:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020248:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774877.020252:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774877.020258:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020262:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774877.020268:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774877.020271:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774877.020275:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774877.020277:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020279:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774877.020282:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774877.020284:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774877.020289:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774877.020293:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774877.020301:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099555840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774877.020311:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774877.020313:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774877.020316:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099555840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774877.020332:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774877.020335:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774877.020341:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774877.020344:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774877.020346:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774877.020351:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774877.020357:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16592 00000400:00000200:1.0:1775774877.020364:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774877.020369:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774877.020374:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774877.020378:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020380:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020384:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020387:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774877.020390:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774877.020392:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774877.020396:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774877.020398:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774877.020400:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774877.020402:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774877.020409:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020412:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020415:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020418:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774877.020423:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774877.020427:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020432:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774877.020434:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774877.020437:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774877.020441:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020444:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774877.020446:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774877.020449:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020452:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774877.020458:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774877.020460:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774877.020464:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774877.020466:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020469:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774877.020470:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774877.020471:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774877.020474:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774877.020477:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774877.020481:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099555968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774877.020494:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774877.020496:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774877.020502:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099555968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774877.020512:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774877.020515:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774877.020520:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774877.020521:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774877.020524:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774877.020527:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774877.020534:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16592 00000400:00000200:1.0:1775774877.020542:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774877.020548:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774877.020551:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774877.020556:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020560:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020563:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020567:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774877.020570:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774877.020572:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774877.020573:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774877.020575:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774877.020576:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774877.020577:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774877.020582:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020585:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020590:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020594:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774877.020598:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774877.020603:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020605:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774877.020607:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774877.020611:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774877.020615:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020617:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774877.020618:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774877.020620:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020622:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774877.020628:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774877.020630:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774877.020632:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774877.020634:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020638:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774877.020640:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774877.020642:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774877.020645:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774877.020650:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774877.020653:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00100000:1.0:1775774877.020655:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099556096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775774877.020661:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:1.0:1775774877.020665:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774877.020668:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774877.020673:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099556096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774877.020684:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774877.020686:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774877.020688:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774877.020689:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774877.020691:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774877.020696:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774877.020702:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16592 00000400:00000200:1.0:1775774877.020711:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774877.020715:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774877.020718:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774877.020722:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020727:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020732:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020735:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774877.020739:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774877.020741:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774877.020745:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774877.020748:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774877.020752:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774877.020757:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774877.020763:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020766:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774877.020769:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020772:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774877.020774:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774877.020818:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774877.020823:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774877.020827:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774877.020829:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774877.020832:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020835:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774877.020837:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774877.020840:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020841:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774877.020846:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774877.020849:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774877.020852:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774877.020855:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774877.020857:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774877.020858:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774877.020861:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774877.020866:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774877.020869:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774877.020875:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099556224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774877.020882:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774877.020884:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774877.020887:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099556224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774877.020895:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774877.020903:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16602) 00000800:00000200:2.1:1775774877.021268:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774877.021271:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774877.021567:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774877.021569:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774877.021759:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774877.021760:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774877.021989:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774877.021991:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000040:1.0:1775774878.299927:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311266038, at 4311266048 00000100:00000010:1.0:1775774878.299943:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775774878.812064:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774878.812070:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774878.812075:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774879.836008:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774879.836011:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774879.836014:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774881.884041:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774881.884044:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774881.884047:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774882.140762:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774882.140773:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774882.141389:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774882.141391:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774882.141576:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774882.141578:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774882.142038:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774882.142042:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774882.142343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774882.142346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775774882.908060:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774882.908064:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774882.908068:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774883.419906:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311271158, at 4311271168 00000100:00000010:1.0:1775774883.419931:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775774884.956032:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774884.956035:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774884.956038:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774885.980012:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774885.980016:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774885.980019:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774887.259923:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774887.259931:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774887.259957:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774887.259960:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774887.259964:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774887.259971:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16602 00000400:00000200:1.0:1775774887.259980:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774887.259985:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774887.259987:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774887.259990:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.259993:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.259995:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.259997:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774887.260000:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774887.260002:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774887.260004:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774887.260006:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774887.260007:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774887.260009:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774887.260011:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260012:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.260014:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260015:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774887.260017:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774887.260021:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260023:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774887.260026:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774887.260030:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774887.260032:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260035:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774887.260037:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774887.260047:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774887.260052:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774887.260056:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774887.260058:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260061:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774887.260063:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774887.260068:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774887.260077:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774887.260084:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774887.260093:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099556992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774887.260105:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774887.260109:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774887.260112:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099556992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774887.260134:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774887.260136:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774887.260139:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774887.260140:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774887.260141:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774887.260144:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774887.260145:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16602 00000400:00000200:1.0:1775774887.260149:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774887.260151:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774887.260153:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774887.260154:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260156:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.260158:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260160:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774887.260162:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774887.260164:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774887.260165:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774887.260166:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774887.260167:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774887.260169:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774887.260171:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260172:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.260174:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260175:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774887.260178:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774887.260180:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260182:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774887.260184:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774887.260189:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774887.260191:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260192:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774887.260195:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774887.260199:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260201:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774887.260208:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774887.260213:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774887.260218:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774887.260220:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260222:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774887.260223:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774887.260225:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774887.260233:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774887.260242:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774887.260257:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099557120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774887.260274:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774887.260279:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774887.260289:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099557120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774887.260320:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774887.260326:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774887.260334:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774887.260336:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774887.260339:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774887.260343:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774887.260350:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16602 00000400:00000200:1.0:1775774887.260357:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774887.260363:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774887.260366:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774887.260373:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260375:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.260376:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260378:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774887.260380:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774887.260383:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774887.260385:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774887.260386:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774887.260388:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774887.260390:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774887.260392:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260396:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.260402:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260409:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774887.260412:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774887.260419:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260430:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774887.260439:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774887.260445:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774887.260449:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260450:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774887.260451:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774887.260458:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260462:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774887.260466:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774887.260473:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774887.260477:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774887.260481:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260483:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774887.260488:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774887.260490:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774887.260494:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774887.260499:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774887.260506:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099557248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774887.260526:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774887.260532:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774887.260549:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099557248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774887.260596:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774887.260605:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774887.260611:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774887.260612:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774887.260615:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774887.260617:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774887.260623:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16602 00000400:00000200:1.0:1775774887.260642:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774887.260649:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774887.260656:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774887.260660:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260666:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.260670:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260676:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774887.260686:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774887.260688:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000800:00000200:2.1:1775774887.260691:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775774887.260695:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.1:1775774887.260696:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000040:1.0:1775774887.260703:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774887.260714:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774887.260724:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774887.260729:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260736:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.260740:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260745:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774887.260748:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774887.260751:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774887.260754:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774887.260760:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774887.260763:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774887.260766:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260768:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774887.260771:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774887.260775:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260808:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774887.260812:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774887.260815:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774887.260817:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774887.260819:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.260821:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774887.260822:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774887.260824:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774887.260827:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774887.260830:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774887.260836:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099557376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774887.260892:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774887.260894:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774887.260900:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099557376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774887.260948:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774887.260953:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774887.260961:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774887.260963:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774887.260964:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774887.260967:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774887.260974:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16602 00000400:00000200:1.0:1775774887.260994:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774887.260998:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774887.260999:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774887.261002:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.261004:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.261006:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.261008:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774887.261010:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774887.261011:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774887.261012:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774887.261014:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774887.261015:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774887.261017:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774887.261019:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.261022:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774887.261028:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774887.261033:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774887.261037:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774887.261041:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774887.261044:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774887.261051:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774887.261057:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774887.261066:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.261072:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774887.261076:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774887.261085:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.261095:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774887.261105:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774887.261108:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774887.261113:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774887.261115:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774887.261118:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774887.261125:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774887.261133:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774887.261144:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775774887.261149:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774887.261167:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099557504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774887.261208:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774887.261217:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774887.261232:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099557504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774887.261265:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774887.261276:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16612) 00000800:00000200:2.1:1775774887.261486:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774887.261487:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774887.261708:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774887.261710:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774887.261961:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774887.261963:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774887.262151:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774887.262153:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775774888.028034:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774888.028039:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774888.028044:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774888.539901:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311276278, at 4311276288 00000100:00000010:1.0:1775774888.539931:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774889.052003:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774889.052006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774889.052010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774891.100026:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774891.100029:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774891.100033:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774892.124006:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774892.124022:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774892.124026:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774892.379944:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774892.379956:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774892.379959:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774892.379963:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774892.379968:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774892.379976:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16607 00000400:00000200:1.0:1775774892.379989:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774892.379997:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774892.380001:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774892.380006:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380009:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380012:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380014:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774892.380017:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774892.380019:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774892.380022:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774892.380024:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774892.380026:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774892.380027:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774892.380030:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380031:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380032:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380034:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774892.380036:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774892.380039:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380040:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774892.380049:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774892.380052:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774892.380055:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380058:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774892.380060:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774892.380072:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774892.380075:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774892.380079:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774892.380080:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380083:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774892.380086:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774892.380091:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774892.380099:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775774892.380106:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774892.380115:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099557632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774892.380124:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774892.380129:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774892.380133:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099557632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774892.380150:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774892.380153:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774892.380156:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774892.380157:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774892.380158:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774892.380160:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774892.380162:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16607 00000400:00000200:1.0:1775774892.380164:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774892.380166:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774892.380167:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774892.380169:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380171:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380172:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380174:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774892.380176:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774892.380178:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774892.380179:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774892.380181:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774892.380183:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774892.380184:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774892.380186:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380188:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380189:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380191:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774892.380192:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774892.380194:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380196:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774892.380197:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774892.380202:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774892.380205:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380208:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774892.380213:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774892.380217:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380219:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774892.380224:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774892.380229:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774892.380234:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774892.380235:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380237:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774892.380240:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774892.380241:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774892.380247:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775774892.380253:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774892.380264:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099557760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774892.380276:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774892.380282:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774892.380289:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099557760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774892.380304:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774892.380308:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774892.380316:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774892.380320:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774892.380323:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774892.380327:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774892.380332:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16607 00000400:00000200:1.0:1775774892.380342:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774892.380348:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774892.380353:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774892.380361:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380365:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380372:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380376:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774892.380378:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774892.380381:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774892.380385:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774892.380387:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774892.380392:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774892.380396:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774892.380404:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380408:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380414:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380419:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774892.380424:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774892.380429:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380435:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774892.380438:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774892.380440:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774892.380442:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380446:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774892.380450:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774892.380455:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380457:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774892.380461:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774892.380464:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774892.380465:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774892.380466:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380470:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774892.380471:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774892.380473:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774892.380480:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774892.380484:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774892.380491:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099557888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775774892.380491:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774892.380497:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1775774892.380506:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774892.380511:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774892.380517:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099557888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774892.380528:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774892.380533:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774892.380536:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774892.380537:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774892.380539:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774892.380542:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774892.380546:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16607 00000400:00000200:1.0:1775774892.380553:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774892.380561:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774892.380566:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774892.380572:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380574:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380576:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380580:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774892.380581:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774892.380583:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774892.380585:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774892.380588:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774892.380591:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774892.380595:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774892.380602:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380605:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380608:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380611:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774892.380617:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774892.380622:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380626:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774892.380627:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774892.380630:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774892.380632:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380635:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774892.380637:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774892.380639:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380640:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774892.380642:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774892.380646:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774892.380649:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774892.380650:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380653:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774892.380655:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774892.380658:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774892.380661:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774892.380665:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774892.380669:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099558016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774892.380678:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774892.380679:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774892.380683:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099558016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774892.380691:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774892.380693:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774892.380695:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774892.380698:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774892.380700:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774892.380701:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774892.380704:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16607 00000400:00000200:1.0:1775774892.380708:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774892.380712:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774892.380715:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774892.380718:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380722:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380725:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380729:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774892.380732:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774892.380735:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774892.380737:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774892.380738:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774892.380741:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774892.380743:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774892.380750:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380754:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774892.380758:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380762:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774892.380766:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774892.380770:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774892.380772:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774892.380773:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774892.380776:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774892.380800:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380802:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774892.380803:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774892.380804:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380805:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774892.380808:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774892.380810:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774892.380811:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774892.380812:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774892.380813:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774892.380814:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774892.380816:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774892.380819:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774892.380823:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774892.380829:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099558144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774892.380838:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774892.380842:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774892.380858:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099558144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774892.380866:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774892.380871:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16617) 00000800:00000200:2.1:1775774892.381184:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774892.381187:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774892.381469:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774892.381471:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774892.381653:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774892.381655:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774892.381868:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774892.381869:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000040:1.0:1775774893.659966:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311281397, at 4311281408 00000100:00000010:1.0:1775774893.659984:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774894.172016:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774894.172022:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774894.172026:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774895.196036:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774895.196042:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774895.196045:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774897.244026:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774897.244029:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774897.244032:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774897.500573:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774897.500583:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774897.501579:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774897.501581:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774897.501821:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774897.501823:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774897.502014:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774897.502015:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774897.502260:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774897.502262:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775774898.268046:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774898.268049:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774898.268052:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774898.779850:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311286518, at 4311286528 00000100:00000010:1.0:1775774898.779864:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775774900.316034:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774900.316039:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774900.316043:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774901.340017:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774901.340020:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774901.340022:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774902.619903:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774902.619911:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774902.619914:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774902.619917:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774902.619921:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774902.619926:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16617 00000400:00000200:1.0:1775774902.619934:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774902.619946:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774902.619951:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774902.619953:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.619956:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.619959:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.619960:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774902.619964:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774902.619965:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774902.619967:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774902.619970:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774902.619971:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774902.619973:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774902.619975:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.619976:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.619977:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.619978:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774902.619980:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774902.619983:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774902.619985:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774902.619987:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774902.619989:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774902.619991:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.619993:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774902.619994:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774902.620003:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774902.620005:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774902.620008:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774902.620010:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620012:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774902.620014:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774902.620018:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774902.620024:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774902.620029:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774902.620036:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099558912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774902.620044:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774902.620048:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774902.620052:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099558912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774902.620071:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774902.620073:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774902.620076:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774902.620077:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774902.620078:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774902.620080:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774902.620082:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16617 00000400:00000200:1.0:1775774902.620084:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774902.620086:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774902.620088:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774902.620090:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620091:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.620092:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620093:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774902.620095:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774902.620096:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774902.620097:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774902.620098:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774902.620100:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774902.620101:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774902.620103:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620104:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.620106:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620108:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774902.620110:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774902.620112:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620116:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774902.620117:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774902.620121:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774902.620126:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620130:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774902.620134:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774902.620138:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620143:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774902.620148:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774902.620152:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774902.620155:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774902.620156:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620159:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774902.620160:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774902.620163:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774902.620167:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774902.620173:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774902.620179:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099559040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774902.620190:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774902.620194:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774902.620200:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099559040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774902.620208:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774902.620211:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774902.620215:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774902.620217:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774902.620218:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774902.620220:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774902.620223:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16617 00000400:00000200:1.0:1775774902.620229:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774902.620234:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774902.620237:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774902.620241:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620244:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.620246:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620247:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774902.620252:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774902.620253:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774902.620254:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774902.620255:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774902.620257:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774902.620261:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774902.620265:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620268:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.620270:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620274:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774902.620278:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774902.620284:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620288:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774902.620290:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774902.620294:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774902.620298:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620301:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774902.620303:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774902.620307:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620309:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774902.620313:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775774902.620316:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774902.620319:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774902.620320:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620321:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774902.620323:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774902.620324:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774902.620327:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774902.620332:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774902.620337:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099559168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774902.620348:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774902.620351:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774902.620357:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099559168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774902.620367:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774902.620370:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774902.620374:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774902.620378:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774902.620380:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774902.620382:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774902.620387:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16617 00000400:00000200:1.0:1775774902.620395:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774902.620400:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774902.620402:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774902.620408:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620409:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.620411:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620412:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774902.620414:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774902.620417:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774902.620418:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774902.620420:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774902.620423:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774902.620427:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774902.620436:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620441:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.620447:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620451:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774902.620455:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774902.620463:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620470:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774902.620474:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775774902.620475:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1775774902.620478:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 02000000:00000001:1.0:1775774902.620480:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774902.620485:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620489:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774902.620493:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774902.620498:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620502:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774902.620508:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774902.620512:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774902.620517:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774902.620520:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620523:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774902.620525:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774902.620527:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774902.620533:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774902.620538:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774902.620547:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099559296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774902.620578:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774902.620582:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774902.620589:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099559296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774902.620604:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774902.620609:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774902.620611:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774902.620614:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774902.620616:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774902.620618:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774902.620620:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16617 00000400:00000200:1.0:1775774902.620626:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774902.620629:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774902.620633:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774902.620637:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620640:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.620643:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620646:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774902.620649:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774902.620651:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774902.620654:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774902.620655:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774902.620658:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774902.620664:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774902.620684:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620686:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774902.620688:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620692:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774902.620694:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774902.620701:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774902.620705:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774902.620708:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774902.620711:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774902.620713:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620716:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774902.620719:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774902.620724:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620729:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774902.620733:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774902.620737:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774902.620743:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774902.620745:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774902.620747:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774902.620750:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774902.620752:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774902.620757:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774902.620761:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774902.620770:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099559424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774902.620804:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774902.620806:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774902.620812:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099559424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774902.620845:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774902.620853:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16628) 00000800:00000200:2.1:1775774902.621036:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774902.621038:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775774902.621255:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774902.621257:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775774902.621468:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774902.621469:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775774902.621659:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774902.621661:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000020:00000001:1.0:1775774903.388023:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774903.388026:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774903.388030:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774903.899882:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311291637, at 4311291648 00000100:00000010:1.0:1775774903.899897:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775774904.412036:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774904.412040:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774904.412043:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774906.460040:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774906.460043:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774906.460046:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774907.483958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774907.483962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774907.483965:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774907.739864:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774907.739873:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774907.739876:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774907.739880:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774907.739885:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774907.739893:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16623 00000400:00000200:1.0:1775774907.739903:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774907.739910:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774907.739914:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774907.739917:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.739921:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.739934:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.739938:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774907.739951:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774907.739956:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774907.739958:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774907.739961:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774907.739962:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774907.739964:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774907.739967:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.739968:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.739970:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.739971:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774907.739973:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774907.739976:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774907.739977:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774907.739979:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774907.739982:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774907.739984:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.739985:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774907.739988:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774907.739996:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775774907.739999:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774907.740002:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774907.740003:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.740005:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774907.740008:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774907.740011:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774907.740019:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775774907.740024:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774907.740031:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099559552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774907.740039:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774907.740043:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774907.740046:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099559552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774907.740069:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774907.740071:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774907.740073:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774907.740075:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774907.740076:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774907.740078:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774907.740080:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16623 00000400:00000200:1.0:1775774907.740083:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774907.740085:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774907.740087:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774907.740089:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740091:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.740092:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740093:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774907.740095:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774907.740096:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774907.740097:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774907.740099:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774907.740100:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774907.740102:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774907.740105:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740106:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.740108:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740109:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774907.740110:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774907.740112:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740113:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774907.740115:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774907.740119:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774907.740121:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.740122:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774907.740125:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774907.740128:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.740130:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774907.740134:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775774907.740136:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774907.740137:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774907.740138:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.740139:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774907.740140:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774907.740141:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774907.740144:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775774907.740147:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774907.740153:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099559680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774907.740166:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774907.740170:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774907.740177:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099559680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774907.740236:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774907.740244:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774907.740249:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774907.740256:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774907.740258:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774907.740266:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774907.740274:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16623 00000400:00000200:1.0:1775774907.740291:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774907.740301:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774907.740302:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774907.740308:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740316:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.740322:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740325:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774907.740329:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774907.740341:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774907.740345:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774907.740350:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774907.740356:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774907.740362:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774907.740368:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740374:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.740377:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740379:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774907.740381:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774907.740395:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740407:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774907.740412:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774907.740417:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774907.740426:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.740433:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774907.740438:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774907.740443:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.740448:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774907.740459:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775774907.740465:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774907.740470:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774907.740472:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.740474:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774907.740475:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774907.740482:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774907.740489:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775774907.740493:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774907.740503:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099559808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774907.740540:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774907.740542:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774907.740552:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099559808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774907.740591:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774907.740594:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774907.740596:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774907.740598:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774907.740599:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774907.740602:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774907.740820:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16623 00000400:00000200:1.0:1775774907.740826:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774907.740840:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774907.740843:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774907.740853:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740865:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.740870:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740880:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774907.740885:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774907.740889:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774907.740891:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774907.740898:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774907.740912:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774907.740921:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774907.740930:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740933:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.740939:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.740948:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774907.740952:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774907.740959:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000800:00000200:2.1:1775774907.740965:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775774907.740967:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774907.740975:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1775774907.740975:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 02000000:00000001:1.0:1775774907.740982:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774907.740991:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.741001:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774907.741003:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774907.741007:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.741009:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774907.741013:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775774907.741024:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774907.741030:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774907.741032:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.741036:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774907.741038:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774907.741042:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774907.741048:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774907.741055:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774907.741068:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099559936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774907.741086:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774907.741092:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774907.741105:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099559936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774907.741125:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774907.741130:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774907.741143:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774907.741146:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774907.741151:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774907.741154:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774907.741160:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16623 00000400:00000200:1.0:1775774907.741166:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774907.741173:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774907.741175:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774907.741179:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.741182:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.741189:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.741196:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774907.741199:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774907.741202:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774907.741204:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774907.741209:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774907.741212:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774907.741216:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774907.741226:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.741235:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774907.741246:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774907.741250:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774907.741253:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774907.741262:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774907.741270:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774907.741276:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774907.741284:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774907.741290:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.741300:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774907.741305:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774907.741314:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.741320:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774907.741326:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775774907.741333:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774907.741339:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774907.741343:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774907.741352:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774907.741357:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774907.741361:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774907.741369:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774907.741375:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774907.741383:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099560064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774907.741425:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774907.741433:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774907.741453:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099560064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774907.741533:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774907.741545:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16633) 00000800:00000200:2.1:1775774907.741827:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774907.741829:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775774907.742054:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774907.742055:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775774907.742236:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774907.742238:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775774907.742492:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774907.742493:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000100:00000040:1.0:1775774909.019923:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311296758, at 4311296768 00000100:00000010:1.0:1775774909.019938:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774909.531989:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774909.531995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774909.531999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774910.556114:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774910.556117:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774910.556119:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774912.604042:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774912.604046:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774912.604049:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774912.860764:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774912.861219:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774912.861411:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774912.861613:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774912.861855:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775774913.628127:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774913.628131:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774913.628135:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774914.139896:0:314334:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311301878, at 4311301888 00000100:00000010:1.0:1775774914.139914:0:314334:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775774915.676098:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774915.676101:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774915.676103:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774916.699956:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774916.699960:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774916.699964:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774917.981177:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774917.981182:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774917.981636:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774917.981639:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774917.981996:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774917.981998:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774917.982278:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774917.982280:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774917.982608:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774917.982611:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775774918.747997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774918.748011:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774918.748016:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774919.259869:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311306998, at 4311307008 00000100:00000010:1.0:1775774919.259886:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775774919.771982:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774919.771986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774919.771990:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774921.820037:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774921.820041:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774921.820045:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774922.843990:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774922.843993:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774922.843996:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774923.100451:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774923.100457:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774923.101117:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774923.101120:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774923.101404:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774923.101406:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774923.101666:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774923.101668:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775774923.102038:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774923.102040:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1775774924.379883:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311312117, at 4311312128 00000100:00000010:1.0:1775774924.379900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775774924.891962:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774924.891966:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774924.891969:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774925.915982:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774925.915986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774925.915989:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774927.964066:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774927.964070:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774927.964075:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774928.220438:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774928.220444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774928.221070:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774928.221072:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774928.221290:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774928.221293:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774928.221528:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774928.221530:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775774928.221755:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774928.221757:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775774928.987979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774928.987983:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774928.987986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774929.499895:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311317237, at 4311317248 00000100:00000010:1.0:1775774929.499910:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775774931.036018:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774931.036021:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774931.036024:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774932.059966:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774932.059969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774932.059971:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774933.340466:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774933.340469:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775774933.341206:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774933.341209:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775774933.341465:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774933.341467:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775774933.341724:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774933.341726:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775774933.342096:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774933.342098:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775774934.108006:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774934.108010:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774934.108013:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774934.619878:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311322357, at 4311322368 00000100:00000010:1.0:1775774934.619892:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774935.131940:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774935.131943:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774935.131945:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774937.179934:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774937.179937:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774937.179939:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774938.204037:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774938.204056:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774938.204061:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774938.460720:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774938.460727:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775774938.461532:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774938.461534:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775774938.461831:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774938.461834:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775774938.462109:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774938.462111:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775774938.462369:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774938.462372:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775774939.227918:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774939.227921:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774939.227923:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774939.739925:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311327478, at 4311327488 00000100:00000010:1.0:1775774939.739942:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775774940.251948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774940.251952:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774940.251955:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774941.276258:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774941.276262:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774941.276266:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774942.299960:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774942.299964:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774942.299967:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774943.324013:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774943.324016:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774943.324019:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774943.580626:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774943.580631:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774943.581534:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774943.581536:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774943.581761:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774943.581763:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774943.581998:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774943.581999:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775774943.582216:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774943.582218:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775774944.347981:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774944.347985:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774944.347987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774944.859855:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311332598, at 4311332608 00000100:00000010:1.0:1775774944.859869:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775774945.371918:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774945.371931:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774945.371934:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774946.396020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774946.396024:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774946.396027:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774947.419986:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774947.419990:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774947.419993:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774948.443968:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774948.443971:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774948.443973:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774948.699908:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774948.699916:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774948.699918:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774948.699921:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774948.699924:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774948.699929:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16664 00000400:00000200:1.0:1775774948.699937:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774948.699942:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774948.699944:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774948.699946:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.699948:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.699950:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.699952:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774948.699955:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774948.699956:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774948.699958:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774948.699960:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774948.699961:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774948.699962:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774948.699969:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.699970:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.699971:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.699973:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774948.699975:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774948.699977:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774948.699978:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774948.699982:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774948.699984:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774948.699986:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.699987:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774948.699988:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774948.699996:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775774948.699999:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774948.700002:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774948.700003:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700005:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774948.700007:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774948.700011:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774948.700016:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774948.700021:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774948.700027:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099564672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774948.700034:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774948.700037:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774948.700045:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099564672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774948.700060:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774948.700063:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774948.700065:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774948.700066:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774948.700066:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774948.700068:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774948.700070:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16664 00000400:00000200:1.0:1775774948.700072:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774948.700073:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774948.700074:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774948.700076:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700077:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.700079:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700080:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774948.700081:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774948.700082:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774948.700082:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774948.700084:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774948.700085:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774948.700087:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774948.700089:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700090:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.700092:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700093:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774948.700095:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774948.700096:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700098:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774948.700099:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774948.700103:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774948.700104:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700109:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774948.700113:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774948.700116:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700118:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774948.700120:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775774948.700124:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774948.700128:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774948.700130:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700133:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774948.700135:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774948.700137:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774948.700140:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774948.700143:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774948.700149:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099564800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774948.700161:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774948.700164:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774948.700168:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099564800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774948.700187:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774948.700189:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774948.700192:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774948.700193:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774948.700194:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774948.700197:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774948.700200:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16664 00000400:00000200:1.0:1775774948.700203:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774948.700208:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774948.700210:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774948.700213:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700216:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.700220:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700223:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774948.700226:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774948.700228:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774948.700230:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774948.700232:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774948.700240:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774948.700242:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774948.700247:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700250:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.700252:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700253:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774948.700255:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774948.700259:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700261:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774948.700264:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774948.700267:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774948.700269:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700273:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774948.700274:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774948.700278:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700280:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774948.700283:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775774948.700287:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774948.700291:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774948.700293:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700295:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774948.700297:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774948.700298:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774948.700301:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774948.700305:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774948.700311:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099564928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774948.700318:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774948.700320:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774948.700324:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099564928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774948.700334:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774948.700338:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:1.0:1775774948.700339:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1775774948.700342:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00080000:1.0:1775774948.700345:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774948.700348:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774948.700351:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774948.700355:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774948.700359:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16664 00000400:00000200:1.0:1775774948.700368:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774948.700372:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774948.700375:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774948.700381:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700385:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.700390:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700393:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774948.700398:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774948.700399:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774948.700402:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774948.700404:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774948.700408:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774948.700410:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774948.700413:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700417:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.700421:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700425:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774948.700429:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774948.700433:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700436:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774948.700438:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774948.700442:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774948.700445:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700447:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774948.700451:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774948.700453:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700457:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774948.700461:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775774948.700465:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774948.700467:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774948.700469:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700472:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774948.700475:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774948.700477:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774948.700484:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774948.700490:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774948.700498:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099565056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774948.700512:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774948.700515:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774948.700521:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099565056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774948.700540:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774948.700542:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774948.700548:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774948.700550:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774948.700553:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774948.700556:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774948.700561:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16664 00000400:00000200:1.0:1775774948.700569:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774948.700574:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774948.700578:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774948.700583:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700589:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.700594:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700599:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774948.700604:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774948.700607:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774948.700611:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774948.700615:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774948.700620:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774948.700626:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774948.700633:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700636:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774948.700641:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700645:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774948.700648:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774948.700653:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774948.700656:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774948.700658:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774948.700662:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774948.700665:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700667:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774948.700670:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774948.700673:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700676:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774948.700679:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775774948.700683:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774948.700686:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774948.700688:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774948.700690:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774948.700693:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774948.700694:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774948.700696:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774948.700699:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774948.700706:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099565184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774948.700718:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774948.700721:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774948.700727:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099565184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774948.700743:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774948.700748:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16674) 00000800:00000200:2.1:1775774948.700836:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774948.700838:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774948.701020:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774948.701021:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774948.701168:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774948.701169:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775774948.701368:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774948.701370:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775774949.467988:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774949.467992:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774949.467995:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774949.979885:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311337717, at 4311337728 00000100:00000010:1.0:1775774949.979897:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774950.491975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774950.491978:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774950.491981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774951.516013:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774951.516017:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774951.516020:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774952.540059:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774952.540064:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774952.540068:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774953.563972:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774953.563977:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774953.563980:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774953.820468:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774953.820475:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774953.821168:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774953.821171:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774953.821361:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774953.821363:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774953.821649:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774953.821651:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775774953.821870:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774953.821872:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775774954.588015:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774954.588019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774954.588023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774955.099880:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311342837, at 4311342848 00000100:00000010:1.0:1775774955.099900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775774955.612029:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774955.612033:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774955.612036:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774956.636028:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774956.636032:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774956.636035:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774957.660029:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774957.660034:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774957.660038:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774958.684016:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774958.684021:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774958.684024:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774958.940632:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774958.940637:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774958.941156:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774958.941158:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774958.941347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774958.941348:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774958.941528:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774958.941530:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775774958.941723:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774958.941725:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775774959.708034:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774959.708038:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774959.708042:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774960.219902:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311347958, at 4311347968 00000100:00000010:1.0:1775774960.219926:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775774960.732101:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774960.732107:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774960.732110:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774961.755999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774961.756003:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774961.756006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774962.780082:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774962.780088:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774962.780092:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774963.804085:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774963.804089:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774963.804092:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774964.060590:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774964.060619:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774964.061419:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774964.061421:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774964.061661:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774964.061663:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774964.061871:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774964.061873:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775774964.062054:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774964.062056:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775774964.828030:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774964.828035:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774964.828038:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774965.339961:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311353078, at 4311353088 00000100:00000010:1.0:1775774965.339988:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775774965.851958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774965.851962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774965.851965:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774966.876017:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774966.876020:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774966.876023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774967.900090:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774967.900095:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774967.900099:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774968.924038:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774968.924042:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774968.924045:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774969.180652:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774969.180660:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774969.181560:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774969.181562:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774969.181867:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774969.181870:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774969.182085:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774969.182086:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775774969.182329:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774969.182331:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775774969.948044:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774969.948048:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774969.948052:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774970.459883:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311358198, at 4311358208 00000100:00000010:1.0:1775774970.459897:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774970.972054:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774970.972059:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774970.972062:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774971.996038:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774971.996041:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774971.996044:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774973.020024:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774973.020029:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774973.020033:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774974.044036:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774974.044042:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774974.044046:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774974.300596:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774974.300605:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775774974.301488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774974.301490:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775774974.301715:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774974.301716:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775774974.302000:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774974.302002:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775774974.302243:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774974.302245:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775774975.067950:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774975.067954:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774975.067958:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774975.579865:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311363318, at 4311363328 00000100:00000010:1.0:1775774975.579880:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775774976.091991:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774976.091995:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774976.091998:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774977.116002:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774977.116007:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774977.116011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774978.140009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774978.140014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774978.140018:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774979.164004:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774979.164006:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774979.164009:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774979.420668:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774979.421229:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774979.421416:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774979.421591:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774979.421768:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775774980.188034:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774980.188038:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774980.188041:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774980.699924:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311368437, at 4311368448 00000100:00000010:1.0:1775774980.699940:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775774981.211992:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774981.211996:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774981.211999:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774982.236011:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774982.236015:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774982.236018:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774983.259990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774983.259995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774983.259998:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774984.540645:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774984.540652:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774984.541455:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774984.541458:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774984.541849:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774984.541852:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774984.542136:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774984.542138:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775774984.542352:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774984.542354:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775774985.308018:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774985.308023:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774985.308027:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774985.819893:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311373558, at 4311373568 00000100:00000010:1.0:1775774985.819920:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775774986.332175:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774986.332183:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774986.332250:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774988.379986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774988.379991:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774988.379994:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774989.404044:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774989.404050:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774989.404054:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774989.659897:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774989.659908:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774989.659911:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774989.659915:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774989.659920:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774989.659928:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16704 00000400:00000200:1.0:1775774989.659938:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774989.659945:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774989.659948:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774989.659952:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.659956:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.659958:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.659960:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774989.659964:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774989.659965:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774989.659967:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774989.659970:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774989.659971:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774989.659973:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774989.659975:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.659976:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.659978:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.659979:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774989.659981:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774989.659983:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774989.659985:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774989.659987:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774989.659989:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774989.659991:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.659993:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774989.659994:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774989.660002:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774989.660005:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774989.660008:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774989.660009:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660012:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774989.660014:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774989.660017:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774989.660025:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775774989.660030:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774989.660037:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099569792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774989.660046:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774989.660049:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774989.660053:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099569792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774989.660074:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774989.660077:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774989.660079:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774989.660080:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774989.660081:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774989.660083:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774989.660085:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16704 00000400:00000200:1.0:1775774989.660088:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774989.660090:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774989.660091:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774989.660093:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660095:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.660096:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660099:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774989.660100:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774989.660102:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774989.660103:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774989.660105:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774989.660106:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774989.660108:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774989.660111:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660112:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.660114:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660115:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774989.660116:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774989.660120:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660124:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774989.660126:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774989.660132:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774989.660139:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660144:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774989.660160:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774989.660167:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660175:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774989.660188:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775774989.660196:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774989.660207:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774989.660209:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660212:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774989.660214:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774989.660218:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774989.660229:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775774989.660235:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774989.660244:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099569920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774989.660253:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774989.660256:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774989.660268:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099569920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774989.660276:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774989.660279:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774989.660283:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774989.660285:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774989.660288:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774989.660294:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774989.660296:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16704 00000400:00000200:1.0:1775774989.660305:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774989.660313:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774989.660314:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774989.660317:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660325:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.660328:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660329:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774989.660332:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774989.660336:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774989.660339:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774989.660340:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774989.660342:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774989.660344:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774989.660363:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660367:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.660372:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660377:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774989.660390:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774989.660402:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660410:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774989.660417:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774989.660421:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774989.660425:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660428:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774989.660431:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774989.660435:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660437:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774989.660440:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774989.660458:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774989.660460:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774989.660462:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660463:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774989.660464:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774989.660471:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774989.660477:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775774989.660485:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774989.660492:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099570048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774989.660520:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774989.660528:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774989.660541:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099570048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774989.660561:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774989.660564:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774989.660580:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774989.660581:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774989.660582:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774989.660584:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774989.660587:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16704 00000400:00000200:1.0:1775774989.660604:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774989.660609:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774989.660611:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774989.660619:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660629:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.660639:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775774989.660640:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774989.660648:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000100:00000001:1.0:1775774989.660649:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774989.660658:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774989.660663:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774989.660667:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774989.660669:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774989.660673:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774989.660681:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774989.660686:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660691:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.660696:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660699:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774989.660702:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774989.660708:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660716:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774989.660724:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774989.660729:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774989.660735:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660740:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774989.660743:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774989.660745:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660747:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774989.660752:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774989.660758:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774989.660765:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774989.660770:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.660833:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774989.660836:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774989.660838:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774989.660843:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775774989.660847:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774989.660852:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099570176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774989.660859:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774989.660863:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774989.660867:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099570176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774989.660889:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774989.660893:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774989.660902:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774989.660904:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774989.660905:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774989.660913:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774989.660917:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16704 00000400:00000200:1.0:1775774989.660924:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774989.660935:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774989.660958:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774989.660964:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660970:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.660977:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.660983:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774989.660989:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774989.660995:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774989.661000:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774989.661003:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774989.661009:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774989.661020:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774989.661034:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.661041:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774989.661051:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774989.661056:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774989.661062:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774989.661070:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774989.661076:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774989.661080:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774989.661086:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774989.661091:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.661094:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774989.661096:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774989.661104:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.661110:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774989.661122:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774989.661129:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774989.661138:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774989.661143:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774989.661159:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774989.661161:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774989.661168:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774989.661178:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775774989.661187:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774989.661202:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099570304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774989.661222:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774989.661227:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774989.661236:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099570304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774989.661271:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774989.661286:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16715) 00000800:00000200:2.1:1775774989.661422:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774989.661424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775774989.661618:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774989.661619:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775774989.661873:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774989.661875:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775774989.662067:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774989.662069:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000100:00000040:1.0:1775774990.939910:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311378678, at 4311378688 00000100:00000010:1.0:1775774990.939925:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775774991.452059:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774991.452062:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774991.452066:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775774992.476044:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774992.476047:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774992.476050:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775774994.779858:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774994.779867:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775774994.779869:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774994.779873:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774994.779876:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774994.779882:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16710 00000400:00000200:1.0:1775774994.779890:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774994.779895:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774994.779897:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774994.779900:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.779902:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.779904:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.779906:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774994.779909:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774994.779911:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775774994.779913:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774994.779915:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775774994.779916:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775774994.779918:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775774994.779920:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.779921:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.779922:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.779924:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774994.779926:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775774994.779928:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774994.779930:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774994.779932:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774994.779934:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774994.779944:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.779946:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775774994.779951:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774994.779958:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775774994.779961:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775774994.779964:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774994.779965:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.779967:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774994.779969:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774994.779973:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774994.779981:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775774994.779986:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774994.779994:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099570432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774994.780002:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774994.780005:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774994.780008:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099570432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774994.780027:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774994.780029:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774994.780032:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775774994.780033:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774994.780034:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774994.780036:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774994.780037:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16710 00000400:00000200:1.0:1775774994.780040:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774994.780042:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774994.780043:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774994.780045:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780047:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.780048:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780050:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774994.780051:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774994.780052:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775774994.780053:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774994.780054:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775774994.780055:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775774994.780072:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775774994.780074:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780076:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.780077:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780079:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774994.780081:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775774994.780083:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780084:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774994.780086:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774994.780090:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774994.780092:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780093:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774994.780097:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774994.780099:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780102:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774994.780105:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775774994.780108:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775774994.780110:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774994.780111:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780112:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774994.780116:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774994.780117:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774994.780122:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775774994.780127:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774994.780136:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099570560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774994.780147:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774994.780151:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774994.780159:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099570560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774994.780172:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774994.780176:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774994.780181:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775774994.780183:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774994.780184:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774994.780190:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774994.780194:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16710 00000400:00000200:1.0:1775774994.780200:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774994.780204:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774994.780206:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774994.780210:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780211:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.780213:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780217:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774994.780219:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774994.780220:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775774994.780221:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774994.780222:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775774994.780227:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775774994.780230:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775774994.780235:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780237:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.780241:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780246:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774994.780250:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775774994.780256:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780260:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774994.780266:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774994.780270:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774994.780277:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780282:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774994.780285:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775774994.780291:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780295:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774994.780302:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775774994.780308:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775774994.780315:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774994.780319:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780323:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774994.780326:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774994.780330:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774994.780334:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775774994.780339:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774994.780348:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099570688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774994.780361:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774994.780363:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774994.780366:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099570688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774994.780371:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774994.780373:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774994.780375:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775774994.780376:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774994.780377:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774994.780379:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774994.780381:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16710 00000400:00000200:1.0:1775774994.780384:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774994.780386:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774994.780387:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774994.780389:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780391:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.780392:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780394:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774994.780395:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774994.780396:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775774994.780397:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774994.780399:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775774994.780400:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775774994.780402:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775774994.780403:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780404:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.780405:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780407:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774994.780408:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775774994.780409:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780412:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774994.780416:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774994.780419:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774994.780422:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780424:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774994.780427:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774994.780433:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780436:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775774994.780436:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000010:1.0:1775774994.780441:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000800:00000200:2.1:1775774994.780441:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000020:00000040:1.0:1775774994.780447:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775774994.780451:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774994.780453:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780457:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774994.780458:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774994.780459:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774994.780466:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775774994.780471:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774994.780477:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099570816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774994.780487:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774994.780491:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774994.780500:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099570816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774994.780512:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775774994.780515:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775774994.780522:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775774994.780525:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775774994.780527:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775774994.780531:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775774994.780538:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16710 00000400:00000200:1.0:1775774994.780543:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775774994.780548:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775774994.780551:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775774994.780556:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780558:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.780560:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780564:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775774994.780565:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775774994.780566:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775774994.780567:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775774994.780569:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775774994.780572:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775774994.780574:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775774994.780576:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780578:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775774994.780583:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780587:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775774994.780589:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775774994.780593:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775774994.780597:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775774994.780602:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775774994.780608:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775774994.780614:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780619:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775774994.780623:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775774994.780628:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780633:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775774994.780641:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775774994.780646:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775774994.780653:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775774994.780656:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775774994.780660:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775774994.780661:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775774994.780663:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775774994.780666:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775774994.780672:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775774994.780684:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099570944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775774994.780709:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775774994.780715:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774994.780724:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099570944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775774994.780740:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775774994.780745:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16720) 00000800:00000200:2.1:1775774994.780949:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774994.780950:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775774994.781151:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774994.781153:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775774994.781327:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774994.781328:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775774994.781515:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774994.781517:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000020:00000001:1.0:1775774995.547992:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774995.547997:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774995.548001:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775774996.059873:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311383797, at 4311383808 00000100:00000010:1.0:1775774996.059887:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775774998.619982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775774998.619986:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775774998.619989:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775774999.900440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774999.900444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774999.901138:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774999.901139:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774999.901332:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774999.901333:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774999.901501:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774999.901503:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775774999.901738:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775774999.901739:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000100:00000040:1.0:1775775001.179901:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311388917, at 4311388928 00000100:00000010:1.0:1775775001.179907:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000800:00000200:2.1:1775775005.020499:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775005.020502:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775005.021030:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775005.021032:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775005.021215:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775005.021217:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775005.021464:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775005.021466:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775005.021676:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775005.021677:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000040:1.0:1775775006.299890:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311394037, at 4311394048 00000100:00000010:1.0:1775775006.299898:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000800:00000200:2.1:1775775010.140471:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775010.140476:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775010.141110:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775010.141113:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775010.141352:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775010.141354:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775010.141574:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775010.141576:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775010.141777:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775010.141824:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000040:1.0:1775775011.419889:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311399157, at 4311399168 00000100:00000010:1.0:1775775011.419898:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000800:00000200:2.1:1775775015.260780:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775015.260812:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775015.262994:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775015.262997:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775015.263445:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775015.263448:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775015.263850:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775015.263853:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775015.264252:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775015.264255:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775775016.027969:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775016.027973:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775016.027977:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775016.539894:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311404279, at 4311404288 00000100:00000010:1.0:1775775016.539902:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775017.052072:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775017.052077:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775017.052080:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775018.075967:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775018.075972:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775018.075976:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775019.099988:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775019.099992:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775019.099996:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775020.123975:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775020.123979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775020.123981:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775020.379920:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775020.379931:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775020.379934:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775020.379938:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775020.379943:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775020.379951:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16735 00000400:00000200:1.0:1775775020.379962:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775020.379969:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775020.379973:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775020.379977:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.379981:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.379984:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.379986:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775020.379990:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775020.379992:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775020.379995:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775020.379999:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775020.380001:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775020.380004:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775020.380007:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380009:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.380011:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380014:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775020.380017:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775020.380020:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380023:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775020.380026:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775020.380029:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775020.380033:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380035:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775020.380037:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775020.380047:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775020.380051:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775020.380056:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775020.380057:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380061:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775020.380063:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775020.380067:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775020.380075:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775775020.380081:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775020.380091:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099573632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775020.380109:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775020.380114:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775020.380119:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099573632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775020.380147:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775020.380151:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775020.380154:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775020.380156:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775020.380158:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775020.380161:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775020.380164:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16735 00000400:00000200:1.0:1775775020.380168:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775020.380172:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775020.380175:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775020.380179:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380181:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.380185:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380187:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775020.380191:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775020.380194:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775020.380198:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775020.380201:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775020.380206:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775020.380212:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775020.380221:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380226:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.380231:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380235:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775020.380240:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775020.380255:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380265:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775020.380270:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775020.380277:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775020.380285:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380288:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775020.380295:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775020.380300:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380304:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775020.380309:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775020.380315:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775020.380321:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775020.380324:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380329:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775020.380332:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775020.380335:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775020.380344:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775775020.380349:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775020.380358:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099573760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775020.380387:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775020.380393:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775020.380403:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099573760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775020.380422:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775020.380427:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775020.380434:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775020.380436:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775020.380438:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775020.380443:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775020.380448:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16735 00000400:00000200:1.0:1775775020.380457:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775020.380464:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775020.380469:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1775775020.380469:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775020.380475:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1775775020.380478:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380484:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.380490:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380494:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775020.380500:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775020.380504:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775020.380507:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775020.380511:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775020.380515:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775020.380520:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775020.380527:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380533:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.380538:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380541:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775020.380545:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775020.380551:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380557:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775020.380561:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775020.380566:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775020.380570:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380575:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775020.380579:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775020.380583:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380586:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775020.380592:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775775020.380597:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775020.380602:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775020.380605:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380609:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775020.380623:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775020.380625:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775020.380632:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775775020.380637:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775020.380647:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099573888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775020.380663:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775020.380667:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775020.380676:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099573888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775020.380696:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775020.380701:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775020.380712:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775020.380716:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775020.380720:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775020.380728:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775020.380734:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16735 00000400:00000200:1.0:1775775020.380744:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775020.380751:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775020.380756:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775020.380762:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380766:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.380771:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380775:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775020.380829:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775020.380832:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775020.380837:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775020.380841:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775020.380847:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775020.380853:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775020.380860:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380864:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.380870:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380874:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775020.380881:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775020.380889:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775020.380894:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775020.380898:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775020.380903:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775020.380908:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380913:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775020.380916:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775020.380923:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380927:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775020.380931:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775020.380934:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775020.380938:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775020.380941:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.380944:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775020.380947:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775020.380951:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775020.380956:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775020.380961:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775020.380968:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099574016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775020.380984:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775020.380986:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775020.380990:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099574016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775020.381005:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775020.381009:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775020.381015:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775020.381018:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775020.381021:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775020.381026:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775020.381030:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16735 00000400:00000200:1.0:1775775020.381036:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775020.381042:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775020.381046:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775020.381048:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.381050:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.381055:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.381058:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775020.381062:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775020.381064:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775020.381067:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775020.381070:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775020.381074:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775020.381076:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775020.381082:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.381085:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775020.381089:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775020.381091:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775020.381096:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775020.381098:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775020.381102:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775020.381104:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775020.381106:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775020.381111:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.381114:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775020.381118:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775020.381121:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.381124:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775020.381130:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775020.381134:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775020.381139:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775020.381142:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775020.381146:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775020.381149:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775020.381151:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775020.381157:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775020.381162:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775020.381170:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099574144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775020.381188:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775020.381191:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775020.381198:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099574144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775775020.381210:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775020.381211:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775020.381216:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1775775020.381217:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16745) 00000800:00000200:2.1:1775775020.381457:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775020.381460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775020.381762:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775020.381764:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775020.381960:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775020.381962:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775021.147983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775021.147987:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775021.147991:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775021.659902:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311409397, at 4311409408 00000100:00000010:1.0:1775775021.659910:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775022.171988:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775022.171997:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775022.172001:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775023.195977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775023.195981:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775023.195984:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775024.219971:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775024.219974:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775024.219978:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775025.244010:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775025.244014:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775025.244019:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775025.500513:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775025.500520:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775025.501308:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775025.501310:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775025.501618:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775025.501621:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775025.501940:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775025.501943:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775025.502241:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775025.502244:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775026.268030:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775026.268036:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775026.268039:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775026.779879:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311414517, at 4311414528 00000100:00000010:1.0:1775775026.779891:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775775027.292010:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775027.292014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775027.292019:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775028.316017:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775028.316020:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775028.316023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775029.340031:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775029.340035:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775029.340038:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775030.363988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775030.363991:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775030.363995:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775030.620667:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775030.620674:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775775030.621549:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775030.621551:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775775030.621768:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775030.621770:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775775030.621970:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775030.621972:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775775030.622177:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775030.622179:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000020:00000001:1.0:1775775031.388021:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775031.388025:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775031.388028:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775031.899893:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311419638, at 4311419648 00000100:00000010:1.0:1775775031.899911:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775032.411997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775032.412001:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775032.412011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775033.435980:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775033.435986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775033.435991:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775034.460012:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775034.460016:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775034.460019:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775035.483947:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775035.483950:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775035.483953:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775035.740505:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775035.740512:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775035.741048:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775035.741050:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775035.741245:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775035.741247:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775035.741442:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775035.741444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775035.741661:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775035.741664:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000020:00000001:1.0:1775775036.507994:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775036.507998:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775036.508001:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775037.019892:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311424757, at 4311424768 00000100:00000010:1.0:1775775037.019902:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775037.531966:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775037.531971:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775037.531975:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775038.555952:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775038.555955:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775038.555958:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775039.579958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775039.579962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775039.579964:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775040.603974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775040.603977:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775040.603979:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775040.859913:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775040.859926:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775040.859929:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775040.859932:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775040.859936:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775040.859941:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16756 00000400:00000200:1.0:1775775040.859950:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775040.859955:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775040.859957:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775040.859960:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.859963:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.859964:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.859966:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775040.859969:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775040.859971:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775040.859973:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775040.859975:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775040.859977:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775040.859978:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775040.859980:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.859981:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.859982:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.859983:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775040.859985:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775040.859987:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775040.859989:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775040.859991:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775040.859993:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775040.859995:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.859996:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775040.859998:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775040.860006:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775775040.860008:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775040.860011:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775040.860012:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860014:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775040.860017:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775040.860020:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775040.860026:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775040.860031:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775040.860038:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099576192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775040.860047:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775040.860051:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775040.860056:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099576192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775040.860079:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775040.860082:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775040.860085:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775040.860087:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775040.860088:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775040.860091:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775040.860093:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16756 00000400:00000200:1.0:1775775040.860097:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775040.860100:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775040.860103:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775040.860106:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860108:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.860111:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860113:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775040.860116:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775040.860119:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775040.860123:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775040.860128:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775040.860132:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775040.860137:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775040.860145:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860150:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.860155:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860160:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775040.860166:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775040.860172:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860177:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775040.860181:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775040.860188:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775040.860192:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860196:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775040.860203:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775040.860209:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860213:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775040.860219:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775775040.860223:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775040.860226:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775040.860229:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860234:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775040.860237:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775040.860241:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775040.860247:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775040.860266:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775040.860274:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099576320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775040.860281:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775040.860283:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775040.860287:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099576320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775040.860296:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775040.860298:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775040.860302:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775040.860304:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775040.860305:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775040.860308:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775040.860310:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16756 00000400:00000200:1.0:1775775040.860314:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775040.860317:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775040.860318:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775040.860321:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860322:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.860324:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860325:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775040.860327:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775040.860328:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775040.860329:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775040.860330:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775040.860331:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775040.860332:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775040.860334:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860336:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.860337:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860338:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775040.860340:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775040.860342:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860343:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775040.860344:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775040.860347:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775040.860348:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860349:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775040.860350:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775040.860352:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860353:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775040.860356:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775775040.860358:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775040.860361:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775040.860362:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860364:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775040.860365:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775040.860367:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775040.860371:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775040.860381:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775040.860390:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099576448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775040.860405:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775040.860409:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775040.860416:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099576448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775040.860427:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775040.860431:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775040.860436:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775040.860439:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775040.860441:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775040.860447:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775040.860454:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16756 00000400:00000200:1.0:1775775040.860461:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775040.860469:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775040.860473:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775040.860478:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860483:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.860489:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860494:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775040.860499:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775040.860502:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775040.860504:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775040.860507:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775040.860510:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775040.860512:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775040.860518:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860521:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.860527:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860532:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775040.860535:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775040.860542:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000800:00000200:2.1:1775775040.860547:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775040.860548:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775040.860552:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1775775040.860554:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 02000000:00000001:1.0:1775775040.860557:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775040.860563:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860568:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775040.860572:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775040.860575:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860580:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775040.860585:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775775040.860588:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775040.860591:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775040.860593:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860594:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775040.860597:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775040.860599:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775040.860603:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775040.860607:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775040.860614:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099576576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775040.860633:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775040.860638:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775040.860644:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099576576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775040.860668:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775040.860672:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775040.860678:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775040.860681:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775040.860684:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775040.860689:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775040.860694:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16756 00000400:00000200:1.0:1775775040.860700:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775040.860703:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775040.860707:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775040.860710:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860712:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.860716:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860717:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775040.860719:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775040.860721:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775040.860724:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775040.860726:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775040.860731:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775040.860735:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775040.860743:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860748:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775040.860755:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860761:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775040.860767:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775040.860772:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775040.860814:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775040.860819:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775040.860823:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775040.860827:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860829:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775040.860832:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775040.860837:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860841:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775040.860846:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775775040.860849:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775040.860852:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775040.860854:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775040.860858:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775040.860860:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775040.860863:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775040.860869:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775040.860873:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775040.860881:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099576704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775040.860890:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775040.860893:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775040.860900:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099576704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775040.860913:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775040.860917:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16766) 00000800:00000200:2.1:1775775040.861144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775040.861146:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775040.861396:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775040.861398:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775040.861670:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775040.861672:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775040.861918:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775040.861921:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000020:00000001:1.0:1775775041.628012:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775041.628015:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775041.628018:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775042.139892:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311429877, at 4311429888 00000100:00000010:1.0:1775775042.139899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775775042.651990:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775042.651994:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775042.651998:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775043.676014:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775043.676017:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775043.676020:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775044.699981:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775044.699985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775044.699989:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775045.723954:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775045.723958:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775045.723961:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775045.980523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775045.980986:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775045.981144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775045.981322:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775045.981488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775046.748005:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775046.748009:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775046.748013:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775047.259908:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311434997, at 4311435008 00000100:00000010:1.0:1775775047.259922:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775047.771976:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775047.771979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775047.771981:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775048.796075:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775048.796081:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775048.796084:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775775049.052034:0:301879:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775775049.052079:0:301879:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775775049.052090:0:301879:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775049.052094:0:301879:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775775049.052099:0:301879:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775049.052103:0:301879:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775775049.052108:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775049.052151:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:1.0:1775775049.052156:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775049.052161:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775049.052167:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d7f8 00000400:00000010:1.0:1775775049.052170:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d7f8. 00000100:00000001:1.0:1775775049.052174:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775049.052184:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454ded80 x1862033099576832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052216:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775049.052222:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099576832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052231:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775049.052233:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775775049.052244:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775049.052247:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7500. 00000400:00000200:1.0:1775775049.052251:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775049.052254:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775049.052258:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720da18 00000400:00000010:1.0:1775775049.052260:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720da18. 00000100:00000001:1.0:1775775049.052262:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775049.052267:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dd880 x1862033099576960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052276:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775049.052279:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099576960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052284:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775049.052286:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775775049.052290:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775049.052291:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7100. 00000400:00000200:1.0:1775775049.052294:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775049.052296:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775049.052298:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dd48 00000400:00000010:1.0:1775775049.052299:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dd48. 00000100:00000001:1.0:1775775049.052301:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775049.052306:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dc380 x1862033099577088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052315:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775049.052318:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099577088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052325:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775049.052327:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775775049.052332:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775049.052334:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7600. 00000400:00000200:1.0:1775775049.052337:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775049.052340:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775049.052344:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d4c8 00000400:00000010:1.0:1775775049.052346:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d4c8. 00000100:00000001:1.0:1775775049.052348:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775049.052353:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454ddc00 x1862033099577216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052362:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775049.052365:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099577216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052371:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775049.052373:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775775049.052378:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775049.052381:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:1.0:1775775049.052384:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775049.052387:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775049.052391:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d198 00000400:00000010:1.0:1775775049.052393:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d198. 00000100:00000001:1.0:1775775049.052396:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775049.052402:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dc700 x1862033099577344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052411:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775049.052415:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099577344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775775070 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775049.052420:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775049.052422:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775775049.819984:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775049.819988:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775049.819992:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775050.844036:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775050.844040:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775050.844042:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775051.100517:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775051.100523:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775051.101294:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775051.101296:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775051.101589:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775051.101591:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775051.101923:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775051.101926:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775051.102340:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775051.102342:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775051.867992:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775051.867995:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775051.867998:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775052.379887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311440117, at 4311440128 00000100:00000010:1.0:1775775052.379895:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775775052.891969:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775052.891972:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775052.891976:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775053.916056:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775053.916059:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775053.916062:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775054.939961:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775054.939963:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775054.939966:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775055.964028:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775055.964032:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775055.964034:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775056.220373:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775056.220379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775056.221019:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775056.221022:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775056.221219:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775056.221221:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775056.221479:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775056.221480:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775056.221695:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775056.221697:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775056.987964:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775056.987968:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775056.987985:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775057.499893:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311445237, at 4311445248 00000100:00000010:1.0:1775775057.499907:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775058.012002:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775058.012006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775058.012010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775059.036041:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775059.036044:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775059.036047:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775060.059966:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775060.059970:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775060.059972:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775061.340360:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775061.340367:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775061.341042:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775061.341045:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775061.341323:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775061.341340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775061.341639:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775061.341641:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775061.341971:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775061.341974:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775775062.107980:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775062.107984:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775062.107988:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775062.619925:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311450357, at 4311450368 00000100:00000010:1.0:1775775062.619934:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775775063.131958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775063.131961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775063.131965:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775065.179975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775065.179978:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775065.179981:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775066.205431:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775066.205435:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775066.205439:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775066.460438:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775066.460451:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775066.461066:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775066.461068:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775066.461251:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775066.461253:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775066.461462:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775066.461464:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775066.461671:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775066.461673:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000040:1.0:1775775067.739904:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311455477, at 4311455488 00000100:00000010:1.0:1775775067.739916:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775775068.251966:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775068.251968:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775068.251971:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775069.276015:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775069.276018:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775069.276020:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775071.323982:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775071.323986:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775071.323989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775071.580489:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775071.580497:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775071.581133:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775071.581136:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775071.581398:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775071.581400:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775071.581697:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775071.581699:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775071.582024:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775071.582039:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775775072.347948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775072.347951:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775072.347954:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775072.859897:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311460597, at 4311460608 00000100:00000010:1.0:1775775072.859907:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775073.371941:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775073.371945:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775073.371949:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775074.395987:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775074.395990:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775074.395992:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775075.419987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775075.419992:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775075.419995:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775076.443958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775076.443963:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775076.443967:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775076.700529:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775076.700536:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775076.701133:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775076.701136:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775076.701413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775076.701415:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775076.701650:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775076.701652:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775076.701877:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775076.701880:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775775077.468016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775077.468021:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775077.468025:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775077.979928:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311465717, at 4311465728 00000100:00000010:1.0:1775775077.979939:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775078.491986:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775078.491990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775078.491993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775079.516012:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775079.516016:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775079.516020:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775080.539977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775080.539980:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775080.539983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775081.564034:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775081.564037:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775081.564040:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775081.820531:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775081.820535:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775081.821285:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775081.821290:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775081.821628:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775081.821629:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775081.821980:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775081.821981:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775081.822237:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775081.822238:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775775082.588023:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775082.588027:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775082.588031:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775083.099878:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311470837, at 4311470848 00000100:00000010:1.0:1775775083.099893:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775083.611966:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775083.611970:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775083.611974:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775084.635952:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775084.635957:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775084.635961:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775085.660028:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775085.660033:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775085.660037:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775086.683977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775086.683981:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775086.683985:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775086.940544:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775086.940551:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775086.941589:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775086.941600:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775086.941798:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775086.941799:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775086.942006:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775086.942015:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775086.942179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775086.942180:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775087.707994:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775087.707998:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775087.708002:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775088.219925:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311475957, at 4311475968 00000100:00000010:1.0:1775775088.219942:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775775088.732006:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775088.732009:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775088.732012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775089.755953:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775089.755956:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775089.755958:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775090.779978:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775090.779982:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775090.779986:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775091.804009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775091.804014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775091.804018:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775092.060539:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775092.060544:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775092.061518:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775092.061521:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775092.061862:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775092.061864:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775092.062144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775092.062147:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775092.062582:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775092.062585:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775092.828033:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775092.828036:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775092.828039:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775093.339888:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311481077, at 4311481088 00000100:00000010:1.0:1775775093.339904:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775093.852025:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775093.852031:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775093.852035:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775094.876058:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775094.876062:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775094.876065:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775095.899990:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775095.899993:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775095.899996:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775096.924053:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775096.924057:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775096.924060:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775097.180649:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775097.180659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775097.181594:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775097.181597:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775097.181951:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775097.181954:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775097.182295:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775097.182311:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775097.182639:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775097.182641:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775097.948017:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775097.948021:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775097.948024:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775098.459877:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311486198, at 4311486208 00000100:00000010:1.0:1775775098.459888:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775098.971980:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775098.971984:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775098.971987:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775099.995986:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775099.995990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775099.995993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775101.019959:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775101.019962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775101.019965:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775102.043937:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775102.043940:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775102.043943:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775102.300354:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775102.300360:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775102.300927:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775102.300930:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775102.301152:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775102.301154:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775102.301437:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775102.301439:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775102.301610:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775102.301612:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775775103.067941:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775103.067944:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775103.067946:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775103.579881:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311491317, at 4311491328 00000100:00000010:1.0:1775775103.579892:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775104.091974:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775104.091977:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775104.091979:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775105.115931:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775105.115936:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775105.115939:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775106.139953:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775106.139956:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775106.139958:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775107.163942:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775107.163946:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775107.163950:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775107.420400:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775107.420402:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775107.420918:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775107.420920:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775107.421082:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775107.421084:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775107.421323:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775107.421325:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775107.421564:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775107.421566:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775775108.187945:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775108.187948:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775108.187950:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775108.699887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311496437, at 4311496448 00000100:00000010:1.0:1775775108.699894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775775109.211925:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775109.211928:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775109.211931:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775110.235996:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775110.235999:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775110.236002:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775111.260016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775111.260020:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775111.260024:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775112.284054:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775112.284059:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775112.284063:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775112.540429:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775112.540770:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775112.541122:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775112.541283:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775112.541435:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775113.307956:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775113.307960:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775113.307964:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775113.819887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311501557, at 4311501568 00000100:00000010:1.0:1775775113.819897:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775114.332008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775114.332010:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775114.332012:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775115.355945:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775115.355948:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775115.355952:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775116.379983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775116.379986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775116.379989:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775117.403962:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775117.403966:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775117.403970:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775117.660440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775117.660445:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775117.660994:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775117.660996:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775117.662549:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775117.662552:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775117.662989:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775117.662991:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775117.663219:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775117.663221:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775118.427987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775118.427990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775118.427994:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775118.939905:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311506677, at 4311506688 00000100:00000010:1.0:1775775118.939914:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775119.452023:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775119.452025:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775119.452028:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775120.475990:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775120.475993:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775120.475997:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775121.499959:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775121.499963:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775121.499966:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775122.524007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775122.524010:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775122.524013:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775122.780482:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775122.780497:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775122.781128:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775122.781131:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775122.781434:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775122.781436:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775122.781724:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775122.781726:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775122.782054:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775122.782058:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775123.547950:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775123.547953:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775123.547955:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775124.059889:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311511797, at 4311511808 00000100:00000010:1.0:1775775124.059896:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775775124.572052:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775124.572055:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775124.572057:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775125.595950:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775125.595953:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775125.595955:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775126.620030:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775126.620034:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775126.620037:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775127.644007:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775127.644010:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775127.644014:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775127.900705:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775127.900715:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775127.901645:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775127.901647:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775127.901879:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775127.901881:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775127.902094:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775127.902096:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775127.902273:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775127.902274:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775775128.667987:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775128.667992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775128.667996:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775129.179888:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311516918, at 4311516928 00000100:00000010:1.0:1775775129.179896:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775775129.691977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775129.691982:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775129.691986:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775130.715930:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775130.715934:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775130.715938:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775131.740001:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775131.740004:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775131.740007:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775132.763956:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775132.763960:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775132.763964:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775133.019915:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775133.019926:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775133.019929:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775133.019933:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775133.019938:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775133.019946:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16848 00000400:00000200:1.0:1775775133.019956:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775133.019963:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775133.019966:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775133.019968:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.019971:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.019972:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.019974:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775133.019977:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775133.019979:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775133.019981:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775133.019983:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775133.019984:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775133.019987:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775133.019989:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.019990:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.019992:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.019993:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775133.019995:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775133.019997:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775133.019999:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775133.020001:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775133.020003:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775133.020005:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020007:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775133.020008:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775133.020017:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775775133.020019:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775133.020023:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775133.020024:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020026:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775133.020028:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775133.020032:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775133.020040:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775775133.020045:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775133.020052:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099587712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775133.020060:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775133.020064:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775133.020068:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099587712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775133.020088:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775133.020090:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775133.020093:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775133.020098:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775133.020100:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775133.020102:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775133.020104:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16848 00000400:00000200:1.0:1775775133.020107:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775133.020109:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775133.020111:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775133.020113:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020115:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.020117:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020119:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775133.020120:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775133.020121:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775133.020123:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775133.020124:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775133.020125:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775133.020127:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775133.020131:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020134:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.020137:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020141:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775133.020142:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775133.020147:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020152:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775133.020156:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775133.020162:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775133.020165:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020168:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775133.020172:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775133.020176:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020180:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775133.020184:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775775133.020187:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775133.020190:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775133.020192:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020194:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775133.020197:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775133.020200:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775133.020204:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775775133.020208:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775133.020215:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099587840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775133.020224:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775133.020227:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775133.020233:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099587840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775133.020244:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775133.020247:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775133.020251:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775133.020254:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775133.020256:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775133.020262:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775133.020266:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16848 00000400:00000200:1.0:1775775133.020272:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775133.020277:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775133.020279:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775133.020282:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020285:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.020289:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020292:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775133.020294:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775133.020296:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775133.020297:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775133.020300:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775133.020304:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775133.020307:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775133.020313:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020315:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.020319:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020322:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775133.020324:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775133.020329:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020331:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775133.020332:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775133.020336:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775133.020339:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020342:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775133.020345:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775133.020348:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020352:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775133.020356:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775775133.020360:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775133.020362:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775133.020364:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020367:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775133.020368:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775133.020369:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775133.020372:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775775133.020374:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775133.020382:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099587968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775133.020396:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775133.020400:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775133.020404:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099587968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775133.020413:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775133.020417:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775133.020424:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775133.020427:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775133.020429:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775133.020433:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:2.1:1775775133.020433:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:1.0:1775775133.020438:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16848 00000800:00000200:2.1:1775775133.020438:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000400:00000200:1.0:1775775133.020446:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775133.020451:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775133.020455:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775133.020460:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020466:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.020471:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020475:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775133.020478:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775133.020479:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775133.020481:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775133.020484:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775133.020487:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775133.020490:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775133.020496:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020500:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.020506:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020511:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775133.020517:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775133.020523:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020527:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775133.020531:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775133.020535:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775133.020540:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020544:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775133.020547:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775133.020552:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020557:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775133.020561:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775775133.020565:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775133.020568:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775133.020569:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020573:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775133.020575:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775133.020578:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775133.020583:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775133.020587:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775133.020594:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099588096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775133.020601:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775133.020604:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775133.020606:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099588096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775133.020612:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775133.020613:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775133.020617:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775133.020620:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775133.020623:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775133.020626:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775133.020631:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16848 00000400:00000200:1.0:1775775133.020641:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775133.020664:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775133.020671:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775133.020678:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020682:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.020688:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020694:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775133.020698:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775133.020701:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775133.020705:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775133.020708:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775133.020713:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775133.020718:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775133.020728:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020734:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775133.020739:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020743:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775133.020750:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775133.020757:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775133.020764:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775133.020769:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775133.020773:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775133.020808:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020812:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775133.020815:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775133.020820:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020823:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775133.020828:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775775133.020832:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775133.020837:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775133.020840:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775133.020845:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775133.020847:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775133.020849:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775133.020854:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775133.020860:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775133.020870:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099588224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775133.020881:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775133.020884:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775133.020889:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099588224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775133.020903:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775133.020911:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16858) 00000800:00000200:2.1:1775775133.021019:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775133.021021:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775133.021201:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775133.021202:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775133.021402:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775133.021403:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775133.021615:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775133.021617:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775775133.787974:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775133.787976:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775133.787979:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775134.299872:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311522037, at 4311522048 00000100:00000010:1.0:1775775134.299885:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775134.811970:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775134.811973:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775134.811975:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775135.835968:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775135.835980:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775135.835982:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775136.859984:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775136.859989:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775136.859993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775137.883972:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775137.883979:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775137.883984:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775138.140302:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775138.140317:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775138.140712:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775138.140713:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775138.140972:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775138.140973:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775138.141168:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775138.141169:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775138.141334:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775138.141336:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775775138.907959:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775138.907963:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775138.907968:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775139.419873:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311527157, at 4311527168 00000100:00000010:1.0:1775775139.419886:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775139.931958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775139.931962:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775139.931966:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775140.955947:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775140.955951:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775140.955955:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775141.979960:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775141.979963:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775141.979971:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775143.003933:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775143.003937:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775143.003939:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775143.260400:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775143.260405:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775143.261056:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775143.261059:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775143.261372:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775143.261374:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775143.261686:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775143.261689:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775143.262002:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775143.262005:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000020:00000001:1.0:1775775144.027955:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775144.027958:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775144.027960:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775144.539887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311532277, at 4311532288 00000100:00000010:1.0:1775775144.539895:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775145.051962:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775145.051965:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775145.051967:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775146.075989:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775146.075992:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775146.075995:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775147.099972:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775147.099982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775147.099986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775148.123949:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775148.123952:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775148.123955:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775148.380425:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775148.380429:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775775148.381112:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775148.381114:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775775148.381308:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775148.381309:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775775148.381504:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775148.381506:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775775148.381774:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775148.381776:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1775775149.147987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775149.147990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775149.147992:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775149.659879:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311537397, at 4311537408 00000100:00000010:1.0:1775775149.659885:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503449c0. 00000020:00000001:1.0:1775775150.171945:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775150.171948:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775150.171951:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775151.195964:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775151.195968:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775151.195971:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775152.219947:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775152.219950:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775152.219952:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775153.243997:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775153.244001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775153.244005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775153.500455:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775153.500460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775775153.501045:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775153.501049:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775775153.501524:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775153.501527:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775775153.501818:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775153.501820:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775775153.502000:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775153.502001:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000020:00000001:1.0:1775775154.268001:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775154.268004:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775154.268007:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775154.779963:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311542517, at 4311542528 00000100:00000010:1.0:1775775154.779967:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000020:00000001:1.0:1775775155.292066:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775155.292070:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775155.292074:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775156.315998:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775156.316002:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775156.316006:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775157.340026:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775157.340030:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775157.340034:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775158.363961:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775158.363966:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775158.363971:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775158.620381:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775158.620386:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775158.620990:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775158.620992:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775158.621231:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775158.621233:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775158.621409:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775158.621411:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775158.621643:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775158.621645:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775159.387968:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775159.387973:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775159.387977:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775159.899901:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311547637, at 4311547648 00000100:00000010:1.0:1775775159.899910:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775775160.411963:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775160.411966:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775160.411969:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775161.435998:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775161.436003:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775161.436006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775162.459975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775162.459978:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775162.459981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775163.484011:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775163.484014:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775163.484017:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775163.740542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775163.740547:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775163.740963:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775163.740965:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775163.741385:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775163.741388:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775163.741694:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775163.741697:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775163.742013:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775163.742016:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775164.507945:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775164.507948:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775164.507951:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775165.019885:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311552757, at 4311552768 00000100:00000010:1.0:1775775165.019894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503440c0. 00000020:00000001:1.0:1775775165.532000:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775165.532003:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775165.532007:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775166.555928:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775166.555932:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775166.555936:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775167.580020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775167.580024:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775167.580028:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775168.603971:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775168.603975:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775168.603979:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775168.860480:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775168.860485:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775168.861175:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775168.861178:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775168.861414:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775168.861416:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775168.861661:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775168.861663:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775168.861940:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775168.861943:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775775169.627981:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775169.627985:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775169.627989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775170.139903:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311557877, at 4311557888 00000100:00000010:1.0:1775775170.139914:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000020:00000001:1.0:1775775170.651984:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775170.652003:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775170.652006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775171.676002:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775171.676007:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775171.676012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775172.700009:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775172.700012:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775172.700015:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775173.724007:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775173.724016:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775173.724024:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775173.980289:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775173.980293:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775173.980694:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775173.980696:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775173.980947:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775173.980949:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775173.981128:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775173.981130:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775173.981372:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775173.981374:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775775174.748005:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775174.748007:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775174.748010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775175.259876:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311562997, at 4311563008 00000100:00000010:1.0:1775775175.259894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000020:00000001:1.0:1775775175.772016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775175.772019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775175.772022:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775176.795994:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775176.795997:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775176.796001:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775177.820004:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775177.820007:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775177.820010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775178.844020:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775178.844026:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775178.844030:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775179.099963:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775179.099977:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775179.099981:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775179.099986:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775179.099992:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775179.100001:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16894 00000400:00000200:1.0:1775775179.100014:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775179.100023:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775179.100027:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775179.100031:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100036:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.100039:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100042:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775179.100048:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775179.100050:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775179.100054:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775179.100057:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775179.100060:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775179.100063:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775179.100067:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100069:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.100071:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100073:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775179.100077:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775179.100080:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100082:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775179.100086:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775179.100090:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775179.100093:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.100096:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775179.100098:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775179.100112:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775179.100117:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775179.100123:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775179.100125:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.100129:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775179.100133:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775179.100139:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775179.100149:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775179.100158:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775179.100170:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099593472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775179.100190:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775179.100196:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775179.100201:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099593472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775179.100231:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775179.100235:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775179.100239:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775179.100241:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775179.100243:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775179.100246:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775179.100249:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16894 00000400:00000200:1.0:1775775179.100254:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775179.100259:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775179.100262:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775179.100268:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100271:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.100281:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100286:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775179.100291:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775179.100295:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775179.100299:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775179.100302:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775179.100307:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775179.100315:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775179.100332:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100340:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.100343:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100350:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775179.100358:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775179.100373:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100389:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775179.100395:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775179.100409:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775179.100414:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.100423:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775179.100429:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775179.100436:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.100445:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775179.100453:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775179.100460:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775179.100465:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775179.100468:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.100474:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775179.100478:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775179.100485:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775179.100497:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775179.100507:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775179.100523:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099593600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775179.100544:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775179.100549:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775179.100563:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099593600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775179.100609:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775179.100614:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775179.100631:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775179.100638:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775179.100645:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775179.100654:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775179.100665:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16894 00000800:00000200:3.1:1775775179.100672:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000400:00000200:1.0:1775775179.100680:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775179.100687:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775179.100695:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775179.100706:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100713:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.100718:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100729:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775179.100733:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775179.100736:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775179.100740:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775179.100745:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775179.100764:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775179.100769:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775179.100834:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100853:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.100861:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100869:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775179.100878:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775179.100889:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775179.100901:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775179.100907:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775179.100915:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775179.100921:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.100931:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775179.100937:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775179.100944:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.100945:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775179.100951:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775775179.100956:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775179.100962:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775179.100965:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.100972:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775179.100975:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775179.100980:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775179.100988:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775179.100994:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775179.101008:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099593728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775179.101031:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775179.101034:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775179.101044:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099593728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775179.101074:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775179.101081:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775179.101090:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775179.101094:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775179.101097:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775179.101104:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775179.101111:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16894 00000400:00000200:1.0:1775775179.101123:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775179.101137:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775179.101144:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775179.101151:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101159:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.101168:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101173:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775179.101182:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775179.101187:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775179.101192:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775179.101195:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775179.101200:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775179.101211:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000800:00000200:3.1:1775775179.101219:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775775179.101224:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101229:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.101235:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101240:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775179.101247:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775179.101259:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101266:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775179.101272:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775179.101280:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775179.101287:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.101299:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775179.101304:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775179.101311:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.101316:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775179.101327:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775179.101334:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775179.101343:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775179.101349:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.101356:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775179.101363:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775179.101367:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775179.101377:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775179.101387:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775179.101400:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099593856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775179.101423:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775179.101429:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775179.101445:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099593856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775179.101471:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775179.101482:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775179.101493:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775179.101498:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775179.101502:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775179.101509:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775179.101516:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16894 00000400:00000200:1.0:1775775179.101532:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775179.101544:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775179.101551:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775179.101564:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101571:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.101577:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101585:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775179.101591:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775179.101595:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775179.101599:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775179.101601:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775179.101604:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775179.101607:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775179.101612:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101614:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775179.101616:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101618:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775179.101621:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775179.101624:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775179.101627:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775179.101628:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775179.101634:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775179.101636:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.101638:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775179.101641:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775179.101644:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.101646:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775179.101651:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775179.101654:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775179.101657:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775179.101659:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775179.101661:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775179.101664:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775179.101666:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775179.101671:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775179.101676:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775179.101683:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099593984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775179.101705:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775179.101710:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775179.101722:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099593984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775179.101754:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775179.101765:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16904) 00000800:00000200:3.1:1775775179.101856:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775179.102025:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775179.102162:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775179.867979:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775179.867983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775179.867986:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775180.379869:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311568117, at 4311568128 00000100:00000010:1.0:1775775180.379883:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000020:00000001:1.0:1775775180.891974:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775180.891977:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775180.891980:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775181.915981:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775181.915986:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775181.915989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775182.940008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775182.940012:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775182.940015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775183.964022:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775183.964027:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775183.964030:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775184.220826:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775184.220832:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775184.221563:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775184.221566:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775184.221891:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775184.221894:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775184.222158:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775184.222160:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775184.222387:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775184.222389:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775184.988004:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775184.988008:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775184.988011:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775185.499902:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311573238, at 4311573248 00000100:00000010:1.0:1775775185.499931:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000020:00000001:1.0:1775775186.012020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775186.012023:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775186.012026:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775187.036049:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775187.036053:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775187.036057:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775188.060036:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775188.060039:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775188.060042:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775189.084000:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775189.084004:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775189.084008:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775189.339921:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775189.339936:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775189.339938:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775189.339942:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775189.339947:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775189.339954:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16904 00000400:00000200:1.0:1775775189.339971:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775189.339978:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775189.339981:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775189.339985:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.339996:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.339998:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340001:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775189.340004:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775189.340005:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775189.340007:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775189.340010:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775189.340011:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775189.340013:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775189.340015:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340017:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340018:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340019:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775189.340021:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775189.340023:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340032:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775189.340035:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775189.340037:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775189.340039:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340041:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775189.340042:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775189.340051:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775775189.340053:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775189.340056:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775189.340061:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340064:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775189.340067:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775189.340071:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775189.340080:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775775189.340087:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775189.340109:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099594752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775189.340121:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775189.340125:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775189.340138:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099594752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775189.340158:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775189.340164:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775189.340167:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775189.340168:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775189.340169:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775189.340172:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775189.340173:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16904 00000400:00000200:1.0:1775775189.340176:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775189.340179:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775189.340180:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775189.340183:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340185:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340187:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340193:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775189.340195:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775189.340196:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775189.340198:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775189.340200:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775189.340204:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775189.340207:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775189.340212:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340215:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340222:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340227:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775189.340232:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775189.340237:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340241:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775189.340244:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775189.340250:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775189.340257:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340261:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775189.340265:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775189.340268:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340270:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775189.340275:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775775189.340278:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775189.340280:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775189.340282:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340290:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775189.340291:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775189.340293:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775189.340300:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775775189.340304:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775189.340310:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099594880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775189.340322:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775189.340325:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775189.340329:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099594880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775189.340340:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775189.340343:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775189.340352:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775189.340354:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775189.340356:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775189.340362:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775189.340366:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16904 00000400:00000200:1.0:1775775189.340377:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775189.340382:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775189.340385:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775189.340389:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340391:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340395:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340398:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775189.340399:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775189.340400:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775189.340401:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775189.340403:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775189.340407:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775189.340410:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775189.340422:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340426:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340430:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340432:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775189.340434:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775189.340436:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340438:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775189.340439:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775189.340442:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775189.340445:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340447:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775189.340454:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775189.340460:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340463:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775189.340466:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775775189.340470:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775189.340475:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775189.340477:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340483:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775189.340485:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775189.340487:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775189.340494:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775775189.340498:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775189.340504:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099595008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1775775189.340508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775189.340511:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1775775189.340522:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775189.340526:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775189.340533:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099595008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775189.340550:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775189.340555:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775189.340560:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775189.340562:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775189.340564:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775189.340568:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775189.340576:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16904 00000400:00000200:1.0:1775775189.340583:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775189.340588:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775189.340592:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775189.340598:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340602:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340605:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340606:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775189.340608:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775189.340611:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775189.340617:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775189.340620:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775189.340623:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775189.340627:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775189.340629:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340631:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340636:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340638:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775189.340640:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775189.340645:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340649:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775189.340651:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775189.340655:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775189.340659:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340692:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775189.340695:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775189.340700:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340702:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775189.340706:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775775189.340711:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775189.340716:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775189.340752:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.340754:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775189.340757:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775189.340760:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775189.340769:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775189.340830:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775189.340842:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099595136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775189.340857:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775189.340861:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775189.340864:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099595136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775189.340885:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775189.340889:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775189.340897:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775189.340900:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775189.340902:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775189.340910:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775189.340914:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16904 00000400:00000200:1.0:1775775189.340917:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775189.340919:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775189.340921:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775189.340924:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340928:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340931:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340934:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775189.340937:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775189.340944:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775189.340949:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775189.340953:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775189.340958:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775189.340978:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775189.340987:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.340993:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775189.340997:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775189.341002:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775189.341006:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775189.341010:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775189.341013:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775189.341016:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775189.341021:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775189.341036:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.341038:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775189.341040:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775189.341044:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.341048:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775189.341054:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775775189.341058:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775189.341062:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775189.341065:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775189.341070:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775189.341074:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775189.341078:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775775189.341097:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775189.341099:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 02000000:00000010:1.0:1775775189.341107:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775189.341114:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775189.341125:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099595264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775189.341139:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775189.341143:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775189.341164:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099595264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775189.341175:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775189.341180:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16914) 00000800:00000200:3.1:1775775189.341393:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775189.341395:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775189.341624:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775189.341626:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775189.341852:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775189.341854:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775190.108032:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775190.108036:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775190.108040:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775190.619878:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311578357, at 4311578368 00000100:00000010:1.0:1775775190.619896:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344180. 00000020:00000001:1.0:1775775191.131958:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775191.131961:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775191.131964:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775192.156009:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775192.156013:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775192.156017:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775193.179994:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775193.179998:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775193.180002:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775194.203971:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775194.203974:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775194.203976:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775194.460463:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775194.460469:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775194.461192:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775194.461194:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775194.461413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775194.461414:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775194.461668:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775194.461669:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775194.461951:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775194.461954:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775775195.228045:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775195.228050:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775195.228053:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775195.739869:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311583477, at 4311583488 00000100:00000010:1.0:1775775195.739881:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775196.251981:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775196.251985:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775196.251987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775197.275979:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775197.275983:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775197.275985:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775198.299947:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775198.299950:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775198.299953:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775199.324063:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775199.324066:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775199.324069:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775199.580449:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775199.580452:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775199.581091:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775199.581095:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775199.581514:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775199.581517:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775199.581769:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775199.581770:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775199.582002:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775199.582004:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1775775200.347992:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775200.347997:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775200.348000:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775200.859898:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311588597, at 4311588608 00000100:00000010:1.0:1775775200.859916:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775775201.371948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775201.371957:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775201.371959:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775202.395940:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775202.395943:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775202.395946:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775203.419988:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775203.419994:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775203.419997:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775204.443972:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775204.443977:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775204.443981:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775204.700460:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775204.700466:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775204.701061:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775204.701063:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775204.701276:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775204.701278:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775204.701531:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775204.701533:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775204.701725:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775204.701727:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775775205.468003:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775205.468006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775205.468009:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775205.979878:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311593717, at 4311593728 00000100:00000010:1.0:1775775205.979894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775206.491943:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775206.491948:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775206.491951:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775207.515935:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775207.515938:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775207.515942:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775208.539968:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775208.539971:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775208.539974:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775209.563909:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775209.563912:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775209.563915:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775209.820390:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775209.820393:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775209.821001:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775209.821003:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775209.821185:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775209.821186:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775209.821478:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775209.821481:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775209.821908:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775209.821911:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775775210.587928:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775210.587931:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775210.587933:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775211.099868:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311598837, at 4311598848 00000100:00000010:1.0:1775775211.099881:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775211.611997:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775211.612001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775211.612005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775212.635974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775212.635977:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775212.635980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775213.659961:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775213.659964:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775213.659966:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775214.684002:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775214.684007:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775214.684010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775214.939885:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775214.939897:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775214.939901:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775214.939906:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775214.939911:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775214.939918:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16930 00000400:00000200:1.0:1775775214.939930:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775214.939938:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775214.939942:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775214.939946:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.939950:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.939953:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.939956:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775214.939961:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775214.939963:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775214.939967:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775214.939971:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775214.939974:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775214.939976:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775214.939980:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.939982:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.939985:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.939987:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775214.939991:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775214.939995:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775214.939998:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775214.940002:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775214.940005:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775214.940009:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.940012:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775214.940014:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775214.940025:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775775214.940029:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775214.940034:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775214.940035:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.940038:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775214.940041:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775214.940045:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775214.940055:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775214.940062:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775214.940070:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099597952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775214.940079:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775214.940084:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775214.940088:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099597952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775214.940109:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775214.940112:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775214.940115:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775214.940116:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775214.940118:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775214.940120:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775214.940122:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16930 00000400:00000200:1.0:1775775214.940125:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775214.940127:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775214.940129:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775214.940131:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940133:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.940134:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940136:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775214.940137:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775214.940138:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775214.940140:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775214.940141:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775214.940143:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775214.940145:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775214.940147:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940148:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.940150:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940151:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775214.940152:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775214.940154:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940155:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775214.940157:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775214.940163:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775214.940164:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.940166:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775214.940170:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775214.940173:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.940174:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775214.940176:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775775214.940178:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775214.940180:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775214.940180:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.940182:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775214.940184:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775214.940187:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775214.940195:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775214.940203:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775214.940214:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099598080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775214.940291:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775214.940299:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775214.940309:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099598080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775214.940365:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775214.940370:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775214.940378:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775214.940381:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775214.940382:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775214.940384:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775214.940392:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16930 00000400:00000200:1.0:1775775214.940400:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775214.940403:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775214.940410:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775214.940416:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940422:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.940431:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940439:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775214.940448:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775214.940453:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775214.940460:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775214.940468:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775214.940477:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775214.940484:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775214.940497:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940504:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.940511:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940516:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775214.940521:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775214.940533:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940545:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775214.940555:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775214.940566:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775214.940576:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.940579:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775214.940583:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775214.940595:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.940602:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775214.940609:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775775214.940614:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775214.940622:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775214.940625:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.940632:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775214.940635:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.1:1775775214.940638:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775775214.940639:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775214.940644:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 02000000:00000010:1.0:1775775214.940647:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775214.940653:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775214.940666:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099598208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775214.940688:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775214.940695:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775214.940721:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099598208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775214.940767:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775214.940773:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775214.940831:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775214.940834:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775214.940837:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775214.940840:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775214.940848:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16930 00000400:00000200:1.0:1775775214.940857:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775214.940874:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775214.940881:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775214.940890:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940901:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.940913:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940917:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775214.940923:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775214.940931:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775214.940938:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775214.940942:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775214.940949:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775214.940955:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775214.940962:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940971:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.940978:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.940987:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775214.940996:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775214.941007:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775214.941015:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775214.941023:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775214.941030:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775214.941034:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.941040:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775214.941046:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775214.941058:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.941064:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775214.941073:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775775214.941081:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775214.941087:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775214.941089:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.941091:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775214.941095:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775214.941102:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775214.941116:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775214.941128:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775214.941139:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099598336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775214.941169:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775214.941175:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775214.941184:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099598336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775214.941239:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775214.941247:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775214.941259:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775214.941265:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775214.941272:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775214.941279:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775214.941290:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16930 00000400:00000200:1.0:1775775214.941302:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775214.941318:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775214.941325:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775214.941338:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.941347:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.941356:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.941361:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775214.941369:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775214.941371:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775214.941376:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775214.941379:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775214.941387:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775214.941396:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775214.941406:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.941413:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775214.941422:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775214.941432:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775214.941442:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775214.941452:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775214.941460:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775214.941465:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775214.941472:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775214.941479:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.941486:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775214.941491:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775214.941498:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.941509:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775214.941522:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775775214.941533:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775214.941547:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775214.941553:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775214.941560:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775214.941562:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775214.941567:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775214.941578:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775214.941589:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775214.941604:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099598464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775214.941627:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775214.941631:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775214.941641:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099598464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775214.941666:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775214.941678:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16940) 00000800:00000200:2.1:1775775214.941704:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775214.941706:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775214.941922:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775214.941924:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775214.942097:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775214.942099:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775214.942379:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775214.942380:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775775215.707988:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775215.707993:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775215.707997:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775216.219912:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311603958, at 4311603968 00000100:00000010:1.0:1775775216.219932:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775775216.732004:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775216.732010:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775216.732015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775217.755987:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775217.756000:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775217.756007:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775218.779957:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775218.779961:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775218.779965:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775219.803978:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775219.803982:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775219.803986:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775220.060444:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775220.060449:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775220.061073:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775220.061078:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775220.061476:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775220.061477:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775220.061732:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775220.061735:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775220.061964:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775220.061966:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775220.827932:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775220.827934:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775220.827937:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775221.339890:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311609077, at 4311609088 00000100:00000010:1.0:1775775221.339907:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775221.851995:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775221.851998:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775221.852001:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775222.875948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775222.875952:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775222.875956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775223.899998:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775223.900001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775223.900005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775224.924011:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775224.924015:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775224.924019:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775225.180434:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775225.180440:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775225.181185:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775225.181188:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775225.181450:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775225.181451:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775225.181711:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775225.181713:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775225.182010:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775225.182012:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775225.947959:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775225.947964:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775225.947967:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775226.459886:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311614197, at 4311614208 00000100:00000010:1.0:1775775226.459902:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775775226.972041:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775226.972045:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775226.972047:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775227.995950:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775227.995953:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775227.995955:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775229.020011:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775229.020015:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775229.020019:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775230.043938:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775230.043941:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775230.043944:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775230.300416:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775230.300423:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775230.300939:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775230.300941:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775230.301141:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775230.301143:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775230.301313:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775230.301315:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775230.301524:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775230.301525:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775231.067929:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775231.067932:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775231.067947:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775231.579864:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311619317, at 4311619328 00000100:00000010:1.0:1775775231.579886:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775232.091986:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775232.091989:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775232.091992:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775233.115991:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775233.115994:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775233.115997:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775234.139956:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775234.139960:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775234.139963:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775235.163981:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775235.163983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775235.163985:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775235.420494:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775235.420501:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775235.421066:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775235.421068:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775235.421341:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775235.421343:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775235.421614:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775235.421616:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775235.421962:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775235.421965:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775775236.187952:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775236.187956:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775236.187961:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775236.699873:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311624437, at 4311624448 00000100:00000010:1.0:1775775236.699885:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775237.211979:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775237.211982:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775237.211985:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775238.235982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775238.235986:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775238.235988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775239.259991:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775239.259995:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775239.259998:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775240.283952:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775240.283956:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775240.283959:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775240.540523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775240.540529:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775240.541149:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775240.541152:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775240.541370:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775240.541373:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775240.541640:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775240.541643:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775240.541899:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775240.541901:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775775241.308020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775241.308025:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775241.308029:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775241.819908:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311629557, at 4311629568 00000100:00000010:1.0:1775775241.819931:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775242.332021:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775242.332025:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775242.332027:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775243.356042:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775243.356046:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775243.356049:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775244.379968:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775244.379971:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775244.379974:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775245.404053:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775245.404056:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775245.404059:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775245.660575:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775245.661124:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775245.661577:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775245.661892:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775245.662089:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775246.428034:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775246.428037:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775246.428040:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775246.939931:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311634677, at 4311634688 00000100:00000010:1.0:1775775246.939951:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775247.451976:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775247.451980:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775247.451984:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775248.476029:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775248.476035:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775248.476040:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775249.500008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775249.500013:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775249.500017:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775250.524010:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775250.524014:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775250.524017:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775250.780550:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775250.780557:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775250.781303:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775250.781313:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775250.781636:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775250.781639:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775250.781955:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775250.781958:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775250.782263:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775250.782265:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775251.547948:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775251.547953:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775251.547956:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775252.059872:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311639797, at 4311639808 00000100:00000010:1.0:1775775252.059889:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775775252.572072:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775252.572075:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775252.572078:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775253.595972:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775253.595976:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775253.595979:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775254.620086:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775254.620092:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775254.620096:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775255.643998:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775255.644004:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775255.644009:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775255.899874:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775255.899882:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775255.899884:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775255.899886:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775255.899890:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775255.899894:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16971 00000400:00000200:1.0:1775775255.899901:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775255.899907:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775255.899909:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775255.899911:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.899913:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.899914:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.899916:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775255.899918:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775255.899920:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775255.899922:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775255.899924:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775255.899925:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775255.899926:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775255.899928:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.899929:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.899930:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.899931:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775255.899933:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775255.899935:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775255.899936:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775255.899938:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775255.899940:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775255.899942:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.899943:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775255.899945:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775255.899952:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775255.899954:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775255.899957:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775255.899958:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.899960:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775255.899961:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775255.899964:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775255.899970:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775775255.899974:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775255.899988:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099603072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775255.899995:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775255.899998:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775255.900001:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099603072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775255.900019:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775255.900022:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775255.900024:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775255.900026:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775255.900027:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775255.900030:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775255.900032:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16971 00000400:00000200:1.0:1775775255.900036:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775255.900040:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775255.900043:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775255.900045:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900048:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.900052:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900055:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775255.900059:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775255.900061:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775255.900063:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775255.900066:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775255.900072:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775255.900077:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775255.900084:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900088:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.900092:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900097:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775255.900100:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775255.900104:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900110:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775255.900113:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775255.900120:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775255.900124:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900128:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775255.900133:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775255.900136:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900138:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775255.900142:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775775255.900146:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775255.900148:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775255.900151:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900156:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775255.900159:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775255.900162:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775255.900169:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775775255.900175:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775255.900184:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099603200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775255.900193:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775255.900195:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775255.900201:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099603200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775255.900213:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775255.900216:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775255.900222:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775255.900225:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775255.900228:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775255.900231:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775255.900234:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16971 00000400:00000200:1.0:1775775255.900248:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775255.900252:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775255.900254:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775255.900256:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900265:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.900268:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900272:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775255.900273:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775255.900275:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775255.900277:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775255.900280:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775255.900281:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775255.900284:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000800:00000200:2.1:1775775255.900285:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775775255.900287:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900291:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.900292:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775775255.900292:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1775775255.900295:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775255.900298:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775255.900303:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900305:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775255.900308:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775255.900311:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775255.900314:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900316:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775255.900317:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775255.900319:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900322:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775255.900326:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775255.900329:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775255.900333:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775255.900334:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900336:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775255.900337:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775255.900340:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775255.900344:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775775255.900347:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775255.900353:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099603328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775255.900378:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775255.900382:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775255.900389:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099603328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775255.900399:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775255.900401:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775255.900407:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775255.900409:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775255.900411:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775255.900415:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775255.900419:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16971 00000400:00000200:1.0:1775775255.900424:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775255.900429:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775255.900432:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775255.900435:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900438:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.900441:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900443:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775255.900445:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775255.900446:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775255.900447:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775255.900449:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775255.900452:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775255.900453:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775255.900455:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900457:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.900461:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900462:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775255.900463:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775255.900465:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900467:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775255.900468:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775255.900472:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775255.900474:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900477:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775255.900479:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775255.900482:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900484:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775255.900487:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775255.900489:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775255.900492:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775255.900495:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900496:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775255.900497:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775255.900499:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775255.900514:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775255.900519:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775255.900525:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099603456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775255.900533:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775255.900537:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775255.900541:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099603456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775255.900550:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775255.900552:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775255.900558:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775255.900560:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775255.900562:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775255.900566:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775255.900569:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16971 00000400:00000200:1.0:1775775255.900574:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775255.900580:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775255.900583:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775255.900588:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900591:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.900593:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900595:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775255.900599:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775255.900600:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775255.900602:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775255.900604:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775255.900607:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775255.900611:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775255.900620:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900624:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775255.900630:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900634:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775255.900639:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775255.900644:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775255.900648:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775255.900652:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775255.900657:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775255.900660:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900662:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775255.900665:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775255.900668:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900671:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775255.900673:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775255.900676:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775255.900677:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775255.900679:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775255.900681:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775255.900683:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775255.900685:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775255.900689:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775255.900693:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775255.900700:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099603584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775255.900716:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775255.900720:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775255.900727:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099603584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775255.900756:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775255.900761:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16981) 00000800:00000200:2.1:1775775255.900934:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775255.900937:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775255.901207:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775255.901209:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775255.901488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775255.901490:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775255.901759:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775255.901762:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775256.667997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775256.668001:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775256.668005:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775257.179913:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311644917, at 4311644928 00000100:00000010:1.0:1775775257.179937:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775257.691975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775257.691979:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775257.691982:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775258.715941:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775258.715944:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775258.715946:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775259.739931:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775259.739942:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775259.739946:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775260.763944:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775260.763947:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775260.763950:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775261.019861:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775261.019869:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775261.019871:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775261.019874:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775261.019877:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775261.019882:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16976 00000400:00000200:1.0:1775775261.019890:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775261.019896:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775261.019899:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775261.019902:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.019905:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.019908:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.019910:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775261.019913:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775261.019916:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775261.019918:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775261.019921:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775261.019923:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775261.019926:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775261.019929:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.019931:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.019933:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.019935:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775261.019938:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775261.019941:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775261.019943:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775261.019945:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775261.019948:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775261.019949:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.019951:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775261.019952:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775261.019960:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775775261.019962:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775261.019965:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775261.019966:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.019968:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775261.019970:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775261.019974:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775261.019979:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775261.019984:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775261.019990:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099603712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775261.019997:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775261.020000:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775261.020003:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099603712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775261.020022:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775261.020025:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775261.020027:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775261.020028:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775261.020029:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775261.020031:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775261.020032:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16976 00000400:00000200:1.0:1775775261.020035:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775261.020037:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775261.020039:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775261.020041:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020042:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.020044:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020045:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775261.020047:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775261.020048:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775261.020049:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775261.020050:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775261.020052:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775261.020053:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775261.020055:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020057:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.020060:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020061:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775261.020064:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775261.020067:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020069:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775261.020071:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775261.020075:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775261.020079:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020082:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775261.020087:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775261.020091:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020093:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775261.020097:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775775261.020100:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775261.020103:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775261.020105:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020107:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775261.020108:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775261.020110:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775261.020113:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775261.020118:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775261.020124:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099603840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775261.020133:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775261.020137:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775261.020141:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099603840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775261.020149:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775261.020151:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775261.020154:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775261.020155:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775261.020156:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775261.020158:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775261.020160:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16976 00000400:00000200:1.0:1775775261.020165:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775261.020169:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775261.020170:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775261.020173:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020175:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.020176:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020180:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775261.020183:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775261.020186:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775261.020188:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775261.020191:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775261.020195:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775261.020201:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775261.020208:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020212:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.020216:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020219:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775261.020222:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775261.020228:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020230:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775261.020232:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775261.020236:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775261.020239:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020242:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775261.020245:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775261.020248:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020250:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775261.020254:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775775261.020258:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775261.020262:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775261.020264:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020267:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775261.020269:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775261.020271:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775261.020276:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775261.020280:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775261.020287:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099603968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775261.020299:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775261.020302:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775261.020305:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099603968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775261.020314:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775261.020317:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775261.020322:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775261.020323:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775261.020325:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1775775261.020325:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775261.020330:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:2.1:1775775261.020330:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00080000:1.0:1775775261.020334:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16976 00000400:00000200:1.0:1775775261.020341:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775261.020346:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775261.020348:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775261.020352:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020356:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.020359:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020362:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775261.020366:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775261.020367:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775261.020368:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775261.020370:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775261.020372:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775261.020375:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775261.020380:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020383:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.020386:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020389:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775261.020393:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775261.020395:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020398:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775261.020400:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775261.020403:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775261.020406:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020409:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775261.020412:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775261.020428:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020429:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775261.020432:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775775261.020436:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775261.020439:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775261.020440:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020441:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775261.020442:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775261.020444:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775261.020449:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775261.020453:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775261.020458:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099604096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775261.020466:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775261.020468:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775261.020473:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099604096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775261.020481:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775261.020484:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775261.020489:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775261.020491:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775261.020493:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775261.020496:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775261.020499:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16976 00000400:00000200:1.0:1775775261.020507:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775261.020511:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775261.020514:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775261.020516:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020517:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.020519:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020523:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775261.020526:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775261.020528:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775261.020531:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775261.020534:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775261.020538:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775261.020544:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775261.020551:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020556:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775261.020561:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020566:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775261.020570:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775261.020574:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775261.020578:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775261.020581:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775261.020583:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775261.020587:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020591:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775261.020594:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775261.020596:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020599:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775261.020603:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775775261.020607:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775261.020609:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775261.020611:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775261.020614:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775261.020616:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775261.020618:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775261.020624:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775261.020635:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775261.020644:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099604224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775261.020662:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775261.020666:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775261.020673:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099604224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775261.020688:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775261.020694:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16986) 00000800:00000200:2.1:1775775261.020831:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775261.020832:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775261.020994:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775261.020996:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775261.021149:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775261.021151:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775261.021328:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775261.021329:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775775261.787956:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775261.787959:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775261.787963:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775262.299853:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311650037, at 4311650048 00000100:00000010:1.0:1775775262.299866:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775775262.812115:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775262.812118:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775262.812121:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775263.835928:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775263.835932:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775263.835936:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775264.860040:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775264.860043:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775264.860046:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775265.883980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775265.883983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775265.883987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775266.139878:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775266.139887:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775266.139889:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775266.139893:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775266.139898:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775266.139905:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 16981 00000400:00000200:1.0:1775775266.139915:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775266.139921:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775266.139924:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775266.139928:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.139932:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.139935:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.139938:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775266.139941:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775266.139944:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775266.139946:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775266.139949:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775266.139951:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775266.139953:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775266.139956:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.139958:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.139960:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.139962:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775266.139965:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775266.139968:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775266.139970:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775266.139973:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775266.139976:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775266.139979:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.139981:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775266.139983:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775266.139992:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775775266.139995:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775266.139999:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775266.140001:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140003:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775266.140006:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775266.140009:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775266.140017:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775775266.140022:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775266.140030:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099604352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775266.140040:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775266.140045:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775266.140050:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099604352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775266.140070:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775266.140076:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775266.140079:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775266.140080:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775266.140082:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775266.140084:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775266.140087:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16981 00000400:00000200:1.0:1775775266.140091:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775266.140094:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775266.140096:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775266.140099:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140102:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.140105:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140109:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775266.140113:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775266.140116:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775266.140120:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775266.140123:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775266.140126:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775266.140130:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775266.140136:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140140:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.140143:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140146:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775266.140148:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775266.140151:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140156:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775266.140158:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775266.140163:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775266.140168:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140170:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775266.140175:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775266.140179:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140184:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775266.140190:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775775266.140196:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775266.140200:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775266.140202:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140204:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775266.140206:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775266.140209:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775266.140216:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775775266.140221:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775266.140230:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099604480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775266.140252:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775266.140255:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775266.140261:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099604480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775266.140276:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775266.140282:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775266.140289:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775266.140292:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775266.140295:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775266.140299:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775266.140305:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16981 00000400:00000200:1.0:1775775266.140314:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000800:00000200:2.1:1775775266.140340:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775266.140342:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000400:00000200:1.0:1775775266.140358:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775266.140364:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775266.140371:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140375:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.140380:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140386:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775266.140390:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775266.140393:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775266.140396:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775266.140400:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775266.140405:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775266.140408:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775266.140415:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140421:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.140424:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140428:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775266.140440:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775266.140446:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140450:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775266.140454:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775266.140458:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775266.140462:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140464:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775266.140467:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775266.140472:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140476:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775266.140481:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775775266.140485:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775266.140489:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775266.140491:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140493:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775266.140496:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775266.140499:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775266.140505:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775775266.140515:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775266.140523:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099604608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775266.140531:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775266.140534:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775266.140540:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099604608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775266.140554:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775266.140560:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775266.140566:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775266.140570:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775266.140574:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775266.140580:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775266.140585:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16981 00000400:00000200:1.0:1775775266.140594:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775266.140602:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775266.140607:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775266.140614:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140619:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.140624:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140629:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775266.140633:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775266.140635:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775266.140637:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775266.140640:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775266.140643:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775266.140648:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775266.140656:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140660:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.140664:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140669:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775266.140675:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775266.140680:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140685:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775266.140690:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775266.140695:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775266.140704:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140708:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775266.140711:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775266.140716:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140720:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775266.140726:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775775266.140730:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775266.140735:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775266.140738:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140741:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775266.140743:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775266.140746:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775266.140752:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775266.140757:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775266.140765:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099604736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775266.140801:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775266.140805:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775266.140812:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099604736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775266.140831:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775266.140835:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775266.140840:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775266.140844:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775266.140846:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775266.140850:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775266.140854:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 16981 00000400:00000200:1.0:1775775266.140861:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775266.140866:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775266.140870:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775266.140875:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140878:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.140883:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140887:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775266.140892:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775266.140894:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775266.140898:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775266.140900:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775266.140905:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775266.140908:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775266.140915:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140917:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775266.140919:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140921:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775266.140928:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775266.140932:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775266.140935:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775266.140938:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775266.140941:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775775266.140942:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775775266.140945:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775266.140946:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000001:1.0:1775775266.140949:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775266.140951:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775266.140955:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140957:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775266.140961:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775775266.140965:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775266.140967:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775266.140968:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775266.140969:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775266.140969:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775266.140971:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775266.140973:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775266.140976:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775266.140981:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099604864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775266.140991:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775266.140994:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775266.140999:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099604864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775266.141015:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775266.141021:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (16991) 00000800:00000200:2.1:1775775266.141246:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775266.141248:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775266.141559:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775266.141561:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775266.141799:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775266.141800:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775775266.907935:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775266.907939:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775266.907943:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775267.419883:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311655157, at 4311655168 00000100:00000010:1.0:1775775267.419896:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775267.931955:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775267.931958:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775267.931960:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775268.956044:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775268.956048:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775268.956051:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775269.979961:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775269.979965:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775269.979968:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775271.004030:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775271.004034:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775271.004037:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775271.260399:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775271.260402:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775271.260999:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775271.261001:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775271.261280:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775271.261282:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775271.261546:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775271.261548:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775271.261924:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775271.261926:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775775272.027949:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775272.027952:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775272.027956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775272.539925:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311660277, at 4311660288 00000100:00000010:1.0:1775775272.539938:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775273.052014:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775273.052016:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775273.052018:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775274.076046:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775274.076049:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775274.076053:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775275.100008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775275.100011:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775275.100013:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775276.123956:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775276.123959:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775276.123962:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775276.380337:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775276.380341:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775276.380907:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775276.380908:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775276.381098:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775276.381099:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775276.381281:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775276.381283:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775276.381490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775276.381492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000020:00000001:1.0:1775775277.147927:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775277.147929:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775277.147932:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775277.659879:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311665397, at 4311665408 00000100:00000010:1.0:1775775277.659888:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775278.171928:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775278.171932:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775278.171935:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775279.195931:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775279.195934:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775279.195936:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775280.219977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775280.219981:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775280.219985:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775281.244008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775281.244011:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775281.244013:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775281.500365:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775281.500371:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775775281.500769:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775281.500771:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775775281.500977:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775281.500978:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775775281.501153:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775281.501155:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775775281.501353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775281.501354:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1775775282.267937:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775282.267941:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775282.267945:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775282.779900:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311670517, at 4311670528 00000100:00000010:1.0:1775775282.779916:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775283.291936:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775283.291939:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775283.291943:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775284.315937:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775284.315940:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775284.315943:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775285.339929:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775285.339931:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775285.339934:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775286.363938:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775286.363942:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775286.363945:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775286.620396:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775286.620400:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775286.620863:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775286.620865:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775286.621139:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775286.621141:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775286.621392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775286.621394:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775286.621647:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775286.621649:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775287.387994:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775287.387998:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775287.388002:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775287.899871:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311675637, at 4311675648 00000100:00000010:1.0:1775775287.899889:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775775288.411925:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775288.411929:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775288.411932:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775289.436001:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775289.436005:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775289.436009:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775290.459962:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775290.459967:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775290.459971:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775291.483976:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775291.483980:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775291.483983:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775291.740413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775291.740419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775291.741003:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775291.741006:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775291.741207:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775291.741209:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775291.741394:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775291.741396:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775291.741568:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775291.741569:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775292.507957:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775292.507963:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775292.507966:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775293.019921:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311680757, at 4311680768 00000100:00000010:1.0:1775775293.019937:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344180. 00000020:00000001:1.0:1775775293.531998:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775293.532001:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775293.532003:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775294.556000:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775294.556003:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775294.556005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775295.579984:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775295.579987:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775295.579990:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775296.603951:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775296.603954:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775296.603956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775296.860328:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775296.860332:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775296.860957:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775296.860959:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775296.861139:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775296.861141:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775296.861308:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775296.861310:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775296.861538:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775296.861540:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775297.627972:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775297.627976:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775297.627978:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775298.139917:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311685877, at 4311685888 00000100:00000010:1.0:1775775298.139927:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000020:00000001:1.0:1775775298.651946:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775298.651950:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775298.651954:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775299.675937:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775299.675940:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775299.675943:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775300.699937:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775300.699940:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775300.699942:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775301.723945:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775301.723948:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775301.723951:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775301.980402:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775301.980407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775301.981021:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775301.981023:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775301.981213:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775301.981215:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775301.981388:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775301.981390:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775301.981599:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775301.981601:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775775302.747971:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775302.747974:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775302.747977:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775303.259897:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311690997, at 4311691008 00000100:00000010:1.0:1775775303.259905:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000020:00000001:1.0:1775775303.771981:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775303.771986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775303.771989:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775304.796009:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775304.796012:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775304.796015:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775305.819951:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775305.819954:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775305.819957:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775306.843986:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775306.843989:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775306.843992:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775307.100383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775307.100388:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775307.100957:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775307.100958:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775307.101145:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775307.101147:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775307.101332:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775307.101334:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775307.101589:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775307.101591:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775775307.867968:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775307.867971:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775307.867973:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775308.379882:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311696117, at 4311696128 00000100:00000010:1.0:1775775308.379899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000020:00000001:1.0:1775775308.891946:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775308.891949:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775308.891952:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775309.915928:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775309.915933:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775309.915941:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775310.940105:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775310.940109:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775310.940113:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775311.963930:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775311.963935:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775311.963939:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775312.219885:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775312.219896:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775312.219899:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775312.219903:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775312.219907:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775312.219915:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17027 00000400:00000200:1.0:1775775312.219925:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775312.219931:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775312.219935:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775312.219939:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.219943:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.219945:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.219947:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775312.219950:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775312.219952:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775312.219954:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775312.219956:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775312.219958:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775312.219960:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775312.219962:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.219963:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.219964:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.219965:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775312.219968:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775312.219970:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775312.219972:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775312.219974:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775312.219976:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775312.219978:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.219980:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775312.219981:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775312.219990:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775312.219993:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775312.219996:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775312.219997:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220000:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775312.220002:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775312.220005:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775312.220011:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775312.220016:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775312.220023:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099610112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775312.220032:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775312.220035:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775312.220038:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099610112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775312.220059:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775312.220061:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775312.220064:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775312.220065:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775312.220066:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775312.220068:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775312.220070:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17027 00000400:00000200:1.0:1775775312.220073:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775312.220075:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775312.220077:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775312.220079:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220080:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.220082:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220084:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775312.220086:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775312.220087:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775312.220089:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775312.220092:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775312.220094:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775312.220098:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775312.220104:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220108:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.220112:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220116:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775312.220118:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775312.220126:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220132:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775312.220137:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775312.220144:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775312.220148:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220150:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775312.220155:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775312.220162:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220166:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775312.220174:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775312.220178:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775312.220185:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775312.220187:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220190:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775312.220194:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775312.220197:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775312.220206:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775312.220211:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775312.220232:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099610240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775312.220256:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775312.220260:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775312.220269:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099610240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775312.220303:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775312.220309:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775312.220321:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775312.220326:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775312.220329:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775312.220336:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775312.220342:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17027 00000400:00000200:1.0:1775775312.220352:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775312.220360:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775312.220365:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1775775312.220369:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775775312.220372:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220375:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.220376:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220377:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775312.220379:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775312.220380:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775312.220381:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775312.220382:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775312.220384:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775312.220385:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775312.220387:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220388:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.220390:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220394:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775312.220399:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775312.220403:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220407:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775312.220410:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775312.220413:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775312.220415:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220417:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775312.220421:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775312.220424:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220425:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775312.220429:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775312.220432:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775312.220434:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775312.220437:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220439:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775312.220440:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775312.220443:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775312.220448:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775312.220452:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775312.220458:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099610368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775312.220469:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775312.220470:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775312.220473:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099610368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775312.220487:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775312.220490:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775312.220497:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775312.220501:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775312.220506:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775312.220512:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775312.220519:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17027 00000400:00000200:1.0:1775775312.220553:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775312.220561:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775312.220565:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775312.220573:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220577:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.220580:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220584:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775312.220588:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775312.220590:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775312.220594:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775312.220597:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775312.220601:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775312.220606:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775312.220613:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220617:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.220623:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220628:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775312.220633:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775312.220641:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220649:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775312.220656:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775312.220663:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775312.220669:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220675:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775312.220680:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775312.220684:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220688:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775312.220694:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000800:00000200:3.1:1775775312.220699:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:1.0:1775775312.220700:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775312.220705:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775312.220709:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220712:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775312.220715:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775312.220719:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775312.220724:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775312.220729:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775312.220736:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099610496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775312.220754:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775312.220758:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775312.220766:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099610496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775312.220825:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775312.220832:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775312.220839:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775312.220843:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775312.220848:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775312.220853:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775312.220857:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17027 00000400:00000200:1.0:1775775312.220865:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775312.220872:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775312.220877:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775312.220883:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220889:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.220895:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220900:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775312.220904:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775312.220907:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775312.220909:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775312.220913:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775312.220918:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775312.220922:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775312.220929:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220934:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775312.220939:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220944:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775312.220948:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775312.220956:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775312.220960:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775312.220964:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775312.220968:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775312.220973:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220978:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775312.220983:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775312.220988:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.220993:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775312.221000:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775312.221005:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775312.221011:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775312.221014:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775312.221018:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775312.221021:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775312.221026:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775312.221033:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775775312.221040:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775312.221052:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099610624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775312.221066:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775312.221071:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775775312.221078:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775775312.221080:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099610624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775312.221098:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775312.221105:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17037) 00000800:00000200:3.1:1775775312.221355:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775312.221493:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775312.987988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775312.987992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775312.987996:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775313.499876:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311701237, at 4311701248 00000100:00000010:1.0:1775775313.499891:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000020:00000001:1.0:1775775314.011961:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775314.011964:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775314.011968:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775315.036020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775315.036024:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775315.036027:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775316.059998:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775316.060002:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775316.060005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775317.083996:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775317.084001:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775317.084004:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775317.341016:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775317.341020:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775317.341389:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775317.341391:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775317.342732:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775317.342735:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775317.343073:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775317.343075:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775317.343380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775317.343383:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775318.107967:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775318.107970:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775318.107972:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775318.619895:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311706358, at 4311706368 00000100:00000010:1.0:1775775318.619905:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000020:00000001:1.0:1775775319.132002:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775319.132006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775319.132009:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775320.155974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775320.155978:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775320.155989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775321.179990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775321.179995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775321.179999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775322.203971:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775322.203975:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775322.203979:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775322.460618:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775322.460623:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775322.461670:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775322.461671:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775322.461899:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775322.461901:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775322.462062:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775322.462064:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775322.462325:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775322.462327:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775323.228002:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775323.228005:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775323.228007:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775323.739910:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311711478, at 4311711488 00000100:00000010:1.0:1775775323.739919:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000020:00000001:1.0:1775775324.251977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775324.251980:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775324.251983:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775325.276008:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775325.276010:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775325.276012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775326.299993:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775326.299996:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775326.299998:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775327.323917:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775327.323921:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775327.323925:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775327.580391:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775327.580395:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775327.580966:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775327.580967:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775327.581174:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775327.581176:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775327.581397:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775327.581399:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775327.581678:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775327.581679:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000020:00000001:1.0:1775775328.347944:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775328.347947:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775328.347951:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775328.859878:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311716597, at 4311716608 00000100:00000010:1.0:1775775328.859890:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344d20. 00000020:00000001:1.0:1775775329.371926:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775329.371930:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775329.371933:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775330.395951:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775330.395954:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775330.395963:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775331.419925:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775331.419928:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775331.419930:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775332.443965:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775332.443969:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775332.443972:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775332.700460:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775332.700463:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775332.701053:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775332.701056:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775332.701339:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775332.701342:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775332.701581:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775332.701584:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775332.701768:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775332.701769:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1775775333.467955:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775333.467958:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775333.467960:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775333.979890:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311721717, at 4311721728 00000100:00000010:1.0:1775775333.979899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503449c0. 00000020:00000001:1.0:1775775334.491946:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775334.491950:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775334.491953:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775335.515970:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775335.515973:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775335.515975:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775336.539943:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775336.539947:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775336.539950:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775337.564009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775337.564013:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775337.564016:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775337.820275:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775337.820279:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775337.820928:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775337.820931:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775337.821385:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775337.821387:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775337.821653:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775337.821655:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775337.821886:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775337.821887:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775775338.587944:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775338.587948:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775338.587952:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775339.099897:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311726837, at 4311726848 00000100:00000010:1.0:1775775339.099911:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000020:00000001:1.0:1775775339.612029:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775339.612033:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775339.612036:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775340.635957:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775340.635959:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775340.635961:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775341.659923:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775341.659925:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775341.659927:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775342.683953:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775342.683956:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775342.683958:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775342.940424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775342.940428:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775342.941053:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775342.941055:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775342.941241:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775342.941242:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775342.941411:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775342.941413:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775342.941642:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775342.941644:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775775343.707933:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775343.707936:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775343.707939:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775344.219893:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311731957, at 4311731968 00000100:00000010:1.0:1775775344.219901:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775344.732016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775344.732019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775344.732022:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775345.756010:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775345.756013:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775345.756017:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775346.779947:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775346.779951:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775346.779953:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775347.803977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775347.803981:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775347.803984:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775348.060322:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775348.060327:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775348.060992:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775348.060993:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775348.061176:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775348.061178:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775348.061348:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775348.061349:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775348.061624:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775348.061625:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775775348.827971:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775348.827976:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775348.827980:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775349.339858:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311737077, at 4311737088 00000100:00000010:1.0:1775775349.339886:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775349.851977:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775349.851981:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775349.851985:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775350.875947:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775350.875950:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775350.875953:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775351.899985:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775351.899988:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775351.899991:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775352.924026:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775352.924029:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775352.924031:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775353.180440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775353.180444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775353.181087:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775353.181089:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775353.181281:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775353.181283:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775353.181468:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775353.181469:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775353.181712:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775353.181714:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775353.947973:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775353.947977:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775353.947980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775354.459877:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311742197, at 4311742208 00000100:00000010:1.0:1775775354.459887:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775354.972005:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775354.972008:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775354.972011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775355.995993:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775355.995997:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775355.996000:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775357.019980:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775357.019984:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775357.019988:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775358.043956:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775358.043959:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775358.043961:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775358.300442:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775358.300446:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775358.301190:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775358.301192:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775358.301390:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775358.301392:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775358.301590:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775358.301591:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775358.301898:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775358.301899:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775359.067938:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775359.067941:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775359.067945:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775359.579886:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311747317, at 4311747328 00000100:00000010:1.0:1775775359.579894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775775360.091952:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775360.091957:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775360.091960:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775361.115918:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775361.115921:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775361.115924:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775362.140013:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775362.140017:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775362.140020:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775363.164030:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775363.164033:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775363.164036:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775363.420340:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775363.420344:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775363.421048:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775363.421051:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775363.421242:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775363.421243:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775363.421496:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775363.421498:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775363.421941:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775363.421944:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775364.187982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775364.187986:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775364.187989:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775364.700002:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311752437, at 4311752448 00000100:00000010:1.0:1775775364.700011:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344d20. 00000020:00000001:1.0:1775775365.212013:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775365.212018:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775365.212021:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775366.235971:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775366.235976:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775366.235980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775367.259965:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775367.259968:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775367.259971:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775368.283966:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775368.283969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775368.283972:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775368.540446:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775368.540452:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775368.541052:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775368.541053:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775368.541278:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775368.541280:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775368.541527:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775368.541529:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775368.541721:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775368.541723:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775775369.308007:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775369.308012:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775369.308016:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775369.819901:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311757557, at 4311757568 00000100:00000010:1.0:1775775369.819911:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503440c0. 00000020:00000001:1.0:1775775370.331928:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775370.331931:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775370.331934:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775371.355970:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775371.355974:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775371.355978:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775372.379947:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775372.379950:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775372.379953:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775373.403992:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775373.403995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775373.403998:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775373.660507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775373.660515:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775373.661160:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775373.661162:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775373.661379:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775373.661381:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775373.661579:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775373.661581:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775373.661891:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775373.661893:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000020:00000001:1.0:1775775374.427994:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775374.427997:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775374.428000:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775374.939887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311762677, at 4311762688 00000100:00000010:1.0:1775775374.939897:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775375.452064:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775375.452069:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775375.452072:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775376.475969:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775376.475974:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775376.475978:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775377.499958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775377.499962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775377.499966:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775378.523970:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775378.523973:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775378.523975:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775378.780385:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775378.780764:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775378.780985:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775378.781148:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775378.781304:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775379.547954:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775379.547958:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775379.547962:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775380.059930:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311767797, at 4311767808 00000100:00000010:1.0:1775775380.059965:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775380.571963:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775380.571966:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775380.571969:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775381.595986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775381.595990:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775381.595993:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775382.620049:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775382.620053:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775382.620056:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775383.644014:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775383.644019:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775383.644022:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775383.900621:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775383.900629:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775383.901478:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775383.901481:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775383.901877:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775383.901879:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775383.902183:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775383.902186:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775383.902512:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775383.902515:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775384.668077:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775384.668081:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775384.668085:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775385.179880:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311772918, at 4311772928 00000100:00000010:1.0:1775775385.179897:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503447e0. 00000020:00000001:1.0:1775775385.691974:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775385.691979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775385.691984:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775386.716026:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775386.716031:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775386.716036:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775387.739975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775387.739979:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775387.739983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775388.764029:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775388.764034:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775388.764038:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775389.020557:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775389.020563:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775389.021156:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775389.021158:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775389.021447:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775389.021450:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775389.021677:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775389.021679:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775389.022037:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775389.022039:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775389.787976:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775389.787981:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775389.787985:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775390.299896:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311778037, at 4311778048 00000100:00000010:1.0:1775775390.299913:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000020:00000001:1.0:1775775390.812038:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775390.812042:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775390.812045:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775391.836023:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775391.836028:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775391.836031:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775392.860032:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775392.860037:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775392.860041:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775393.884038:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775393.884043:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775393.884048:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775394.140438:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775394.140444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775394.141206:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775394.141209:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775394.141422:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775394.141424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775394.141611:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775394.141613:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775394.141941:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775394.141943:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775775394.908008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775394.908011:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775394.908014:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775395.419883:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311783157, at 4311783168 00000100:00000010:1.0:1775775395.419893:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775395.932011:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775395.932015:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775395.932017:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775396.955983:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775396.955986:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775396.955989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775397.979963:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775397.979966:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775397.979970:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775399.003961:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775399.003964:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775399.003967:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775399.260306:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775399.260313:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775399.260981:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775399.260983:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775399.261189:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775399.261190:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775399.261387:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775399.261389:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775399.261724:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775399.261726:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775775400.027944:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775400.027947:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775400.027950:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775400.539935:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311788277, at 4311788288 00000100:00000010:1.0:1775775400.539950:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000020:00000001:1.0:1775775401.051999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775401.052002:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775401.052005:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775402.075917:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775402.075921:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775402.075923:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775403.099986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775403.099993:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775403.099997:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775404.379864:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775404.379871:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775404.379874:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775404.379877:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775404.379881:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775404.379887:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17119 00000400:00000200:1.0:1775775404.379895:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775404.379900:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775404.379902:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775404.379905:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.379908:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.379909:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.379911:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775404.379925:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775404.379926:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775404.379928:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775404.379931:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775404.379932:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775404.379934:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775404.379937:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.379938:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.379939:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.379940:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775404.379942:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775404.379945:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775404.379946:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775404.379949:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775404.379951:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775404.379953:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.379955:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775404.379956:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775404.379963:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775775404.379966:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775404.379969:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775404.379970:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.379973:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775404.379975:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775404.379978:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775404.379984:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775775404.379989:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775404.379995:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099621632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775404.380003:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775404.380007:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775404.380010:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099621632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775404.380029:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775404.380032:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775404.380034:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775404.380035:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775404.380036:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775404.380038:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775404.380040:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17119 00000400:00000200:1.0:1775775404.380043:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775404.380045:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775404.380046:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775404.380048:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380050:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.380051:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380052:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775404.380053:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775404.380054:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775404.380055:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775404.380057:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775404.380059:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775404.380060:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775404.380062:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380063:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.380065:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380066:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775404.380068:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775404.380071:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380073:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775404.380075:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775404.380083:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775404.380087:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380091:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775404.380096:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775404.380101:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380106:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775404.380112:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775775404.380116:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775404.380119:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775404.380120:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380122:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775404.380125:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775404.380128:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775404.380134:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775775404.380140:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775404.380148:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099621760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775404.380166:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775404.380170:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775404.380175:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099621760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775404.380187:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775404.380190:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775404.380194:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775404.380196:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775404.380198:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775404.380201:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775404.380203:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17119 00000400:00000200:1.0:1775775404.380212:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775404.380219:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775404.380221:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775404.380227:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380231:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.380237:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380241:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775404.380247:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775404.380251:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775404.380255:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775404.380260:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775404.380264:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775404.380269:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775404.380275:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380278:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.380283:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380285:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775404.380286:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775404.380290:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380298:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775404.380299:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775404.380304:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775404.380305:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380307:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775404.380309:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775404.380312:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380315:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775404.380319:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775775404.380323:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775404.380327:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775404.380329:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380332:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775404.380335:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775404.380338:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775404.380344:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775404.380348:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775404.380358:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099621888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775404.380372:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775404.380374:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775404.380378:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099621888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775404.380390:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775404.380393:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775404.380398:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:3.1:1775775404.380401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775404.380402:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775404.380404:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.1:1775775404.380406:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000100:00000001:1.0:1775775404.380407:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775404.380413:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17119 00000400:00000200:1.0:1775775404.380422:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775404.380426:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775404.380430:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775404.380435:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380439:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.380442:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380446:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775404.380450:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775404.380451:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775404.380453:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775404.380456:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775404.380458:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775404.380460:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775404.380466:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380471:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.380475:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380478:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775404.380480:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775404.380484:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380487:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775404.380491:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775404.380494:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775404.380498:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380502:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775404.380504:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775404.380507:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380509:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775404.380512:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775775404.380516:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775404.380520:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775404.380523:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380525:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775404.380527:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775404.380530:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775404.380534:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775404.380539:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775404.380548:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099622016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775404.380566:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775404.380571:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775404.380578:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099622016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775404.380597:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775404.380602:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775404.380607:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775404.380609:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775404.380611:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775404.380613:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775404.380617:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17119 00000400:00000200:1.0:1775775404.380625:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775404.380629:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775404.380633:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775404.380638:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380640:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.380644:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380645:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775404.380648:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775404.380649:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775404.380652:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775404.380654:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775404.380658:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775404.380661:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775404.380668:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380672:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775404.380677:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380681:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775404.380683:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775404.380685:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775404.380687:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775404.380689:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775404.380694:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775404.380700:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380705:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775404.380708:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775404.380711:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380714:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775404.380722:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775775404.380727:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775404.380732:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775404.380736:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775404.380742:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775404.380744:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775404.380747:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775404.380751:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775404.380755:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775404.380762:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099622144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775404.380771:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775404.380776:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775404.380822:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099622144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775404.380847:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775404.380854:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17129) 00000800:00000200:3.1:1775775404.381010:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775404.381012:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775775404.381198:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775404.381199:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775775404.381371:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775404.381372:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775775404.381626:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775404.381629:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775775405.148018:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775405.148023:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775405.148026:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775405.659915:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311793397, at 4311793408 00000100:00000010:1.0:1775775405.659925:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000020:00000001:1.0:1775775406.172002:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775406.172005:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775406.172008:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775408.220023:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775408.220029:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775408.220034:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775409.243985:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775409.243989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775409.243992:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775409.500576:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775409.500582:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775775409.501180:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775409.501183:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775775409.501380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775409.501382:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775775409.501598:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775409.501600:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775775409.501808:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775409.501811:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000100:00000040:1.0:1775775410.779917:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311798517, at 4311798528 00000100:00000010:1.0:1775775410.779936:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344a80. 00000020:00000001:1.0:1775775411.291988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775411.291994:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775411.292000:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775412.315975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775412.315979:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775412.315982:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775414.363964:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775414.363967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775414.363970:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775414.620852:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775414.620860:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775775414.621432:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775414.621434:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775775414.621759:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775414.621762:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775775414.622105:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775414.622108:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775775414.622432:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775414.622435:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775775415.388050:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775415.388053:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775415.388056:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775415.899946:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311803638, at 4311803648 00000100:00000010:1.0:1775775415.899962:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000020:00000001:1.0:1775775417.435929:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775417.435932:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775417.435935:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775418.460031:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775418.460034:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775418.460037:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775419.740304:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775419.740307:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775419.740727:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775419.740729:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775419.740936:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775419.740937:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775419.741109:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775419.741110:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775419.741276:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775419.741277:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775420.508000:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775420.508004:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775420.508007:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775421.019906:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311808757, at 4311808768 00000100:00000010:1.0:1775775421.019912:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775421.531936:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775421.531940:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775421.531943:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775423.579983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775423.579987:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775423.579990:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775424.603974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775424.603978:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775424.603981:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775424.860569:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775424.860572:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775424.861138:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775424.861140:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775424.861330:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775424.861331:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775424.861523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775424.861525:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775424.861763:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775424.861767:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000040:1.0:1775775426.139923:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311813877, at 4311813888 00000100:00000010:1.0:1775775426.139958:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775426.651931:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775426.651934:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775426.651943:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775427.676042:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775427.676046:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775427.676049:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775429.724020:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775429.724026:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775429.724030:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775429.980664:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775429.980669:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775429.981359:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775429.981363:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775429.981606:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775429.981608:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775429.981858:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775429.981859:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775429.982037:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775429.982038:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775430.747968:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775430.747971:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775430.747974:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775431.259897:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311818998, at 4311819008 00000100:00000010:1.0:1775775431.259907:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775432.796050:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775432.796063:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775432.796066:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775433.819979:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775433.819983:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775433.819986:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775435.100549:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775435.100558:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775435.101149:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775435.101152:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775435.101476:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775435.101479:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775435.101799:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775435.101802:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775435.102095:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775435.102098:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775775435.868039:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775435.868043:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775435.868046:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775436.379934:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311824117, at 4311824128 00000100:00000010:1.0:1775775436.379950:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775775436.892075:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775436.892078:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775436.892082:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775438.939951:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775438.939955:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775438.939958:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775439.964029:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775439.964033:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775439.964036:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775440.220716:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775440.220724:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775440.221802:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775440.221808:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775440.222110:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775440.222112:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775440.222363:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775440.222365:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775440.222617:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775440.222619:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000040:1.0:1775775441.499866:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311829238, at 4311829248 00000100:00000010:1.0:1775775441.499879:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775442.011955:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775442.011958:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775442.011960:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775443.036129:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775443.036133:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775443.036136:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775445.084070:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775445.084074:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775445.084078:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775445.340508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775445.341097:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775445.341600:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775445.341772:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775445.341975:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775446.107965:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775446.107970:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775446.107973:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775446.619940:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311834357, at 4311834368 00000100:00000010:1.0:1775775446.619955:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775448.155935:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775448.155939:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775448.155942:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775449.179984:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775449.179987:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775449.179990:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775450.461016:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775450.461023:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775450.461409:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775450.461412:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775450.461693:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775450.461695:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775450.462027:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775450.462030:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775450.462334:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775450.462344:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775451.228010:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775451.228014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775451.228018:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775451.739867:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311839478, at 4311839488 00000100:00000010:1.0:1775775451.739877:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775452.251961:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775452.251964:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775452.251967:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775454.299971:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775454.299976:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775454.299980:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775455.324069:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775455.324074:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775455.324078:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775455.580524:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775455.580531:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775775455.581209:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775455.581212:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775775455.581441:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775455.581443:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775775455.581629:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775455.581631:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775775455.581843:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775455.581844:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000100:00000040:1.0:1775775456.859861:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311844597, at 4311844608 00000100:00000010:1.0:1775775456.859874:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775775457.371976:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775457.371979:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775457.371982:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775458.395988:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775458.395993:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775458.395997:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775460.443925:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775460.443930:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775460.443946:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775460.700431:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775460.700436:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775775460.701014:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775460.701016:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775775460.701314:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775460.701316:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775775460.701619:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775460.701622:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775775460.701895:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775460.701898:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000020:00000001:1.0:1775775461.468003:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775461.468007:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775461.468010:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775461.979882:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311849717, at 4311849728 00000100:00000010:1.0:1775775461.979901:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775775462.491957:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775462.491960:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775462.491963:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775463.515968:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775463.515973:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775463.515976:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775464.540007:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775464.540011:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775464.540014:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775465.563966:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775465.563969:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775465.563972:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775465.820307:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775465.820311:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775465.820720:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775465.820722:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775465.820988:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775465.820990:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775465.821167:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775465.821169:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775465.821401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775465.821402:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1775775466.587975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775466.587978:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775466.587981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775467.099889:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311854837, at 4311854848 00000100:00000010:1.0:1775775467.099901:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775467.611963:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775467.611966:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775467.611969:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775468.636010:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775468.636013:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775468.636015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775469.660035:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775469.660038:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775469.660041:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775470.683994:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775470.684002:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775470.684008:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775470.940381:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775470.940385:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775470.941054:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775470.941055:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775470.941241:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775470.941242:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775470.941418:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775470.941420:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775470.941684:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775470.941686:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775775471.707986:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775471.707990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775471.707994:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775472.219908:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311859957, at 4311859968 00000100:00000010:1.0:1775775472.219931:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775472.732039:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775472.732043:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775472.732047:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775473.755981:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775473.755984:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775473.755987:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775474.780048:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775474.780053:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775474.780058:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775475.804006:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775475.804010:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775475.804014:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775476.060400:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775476.060406:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775476.060972:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775476.060973:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775476.061156:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775476.061157:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775476.061337:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775476.061339:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775476.061531:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775476.061532:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775775476.827988:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775476.827991:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775476.827995:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775477.339896:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311865077, at 4311865088 00000100:00000010:1.0:1775775477.339910:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775775477.851933:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775477.851938:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775477.851941:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775478.876005:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775478.876007:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775478.876010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775479.899984:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775479.899988:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775479.899992:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775480.924006:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775480.924009:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775480.924012:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775481.180454:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775481.180461:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775481.181051:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775481.181053:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775481.181269:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775481.181270:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775481.181503:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775481.181505:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775481.181761:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775481.181763:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775775481.947979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775481.947983:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775481.947987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775482.459876:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311870197, at 4311870208 00000100:00000010:1.0:1775775482.459900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775775482.971954:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775482.971958:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775482.971962:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775483.995977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775483.995980:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775483.995983:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775485.019935:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775485.019938:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775485.019941:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775486.044051:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775486.044054:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775486.044057:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775486.300404:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775486.300409:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775486.300943:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775486.300944:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775486.301179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775486.301181:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775486.301368:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775486.301370:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775486.301584:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775486.301586:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775487.067910:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775487.067914:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775487.067918:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775487.579886:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311875317, at 4311875328 00000100:00000010:1.0:1775775487.579900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775488.091964:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775488.091967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775488.091970:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775489.115964:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775489.115967:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775489.115970:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775490.139982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775490.139985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775490.139988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775491.164002:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775491.164006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775491.164010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775491.420380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775491.420385:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775491.421013:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775491.421015:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775491.421226:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775491.421228:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775491.421431:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775491.421433:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775491.421607:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775491.421608:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775492.187977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775492.187981:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775492.187983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775492.699893:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311880437, at 4311880448 00000100:00000010:1.0:1775775492.699911:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775493.211999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775493.212004:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775493.212008:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775494.235990:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775494.235993:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775494.235996:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775495.259996:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775495.260002:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775495.260008:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775496.284064:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775496.284068:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775496.284071:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775496.539950:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775496.539961:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775496.539964:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775496.539968:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775496.539972:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775496.539979:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17211 00000400:00000200:1.0:1775775496.539990:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775496.539997:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775496.540000:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775496.540004:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540007:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.540010:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540013:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775496.540017:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775496.540019:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775496.540022:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775496.540025:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775496.540027:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775496.540030:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775496.540033:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540035:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.540037:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540040:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775496.540042:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775496.540046:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540049:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775496.540052:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775496.540055:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775496.540058:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.540061:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775496.540063:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775496.540073:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775775496.540078:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775496.540083:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775496.540085:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.540088:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775496.540103:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775496.540111:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775496.540120:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775496.540127:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775496.540137:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099633152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775496.540148:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775496.540153:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775496.540158:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099633152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775496.540187:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775496.540191:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775496.540195:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775496.540197:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775496.540198:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775496.540201:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775496.540204:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17211 00000400:00000200:1.0:1775775496.540209:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775496.540212:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775496.540216:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775496.540220:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540222:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.540225:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540227:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775496.540230:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775496.540232:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775496.540234:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775496.540237:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775496.540240:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775496.540243:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775496.540246:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540249:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.540253:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540259:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775496.540268:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775496.540276:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540282:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775496.540288:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775496.540300:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775496.540323:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.540329:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775496.540338:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775496.540353:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.540359:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775496.540364:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775775496.540371:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775496.540375:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775496.540378:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.540385:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775496.540389:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775496.540393:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775496.540402:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775496.540408:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775496.540419:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099633280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775496.540457:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775496.540465:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775496.540479:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099633280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775496.540504:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775496.540511:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775496.540520:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775496.540524:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775496.540528:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775496.540534:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775496.540541:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17211 00000400:00000200:1.0:1775775496.540557:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775496.540566:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775496.540576:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775496.540591:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540598:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.540608:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540614:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775496.540618:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775496.540619:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775496.540621:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775496.540627:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775496.540632:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775496.540637:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775496.540644:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540648:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.540651:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.540659:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775496.540666:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775496.540676:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000800:00000200:2.1:1775775496.540676:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775496.540684:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1775775496.540686:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775496.540696:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775496.540708:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775496.540713:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.540716:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775496.540720:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775496.540728:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.540732:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775496.540742:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775775496.540749:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775496.540759:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775496.540763:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.540766:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775496.540769:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775496.540774:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775496.540848:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775775496.540856:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775496.540872:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099633408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775496.540887:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775496.540892:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775496.540905:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099633408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775496.540924:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775496.540930:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775496.540941:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775496.540945:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775496.540947:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775496.540956:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775496.540964:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17211 00000400:00000200:1.0:1775775496.540975:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775496.540984:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775496.540991:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775496.541004:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541009:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.541014:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541019:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775496.541027:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775496.541031:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775496.541035:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775496.541039:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775496.541045:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775496.541057:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775496.541068:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541075:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.541084:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541093:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775496.541098:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775496.541107:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541114:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775496.541121:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775496.541132:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775496.541138:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.541152:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775496.541159:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775496.541169:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.541177:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775496.541190:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775775496.541200:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775496.541211:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775496.541214:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.541223:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775496.541231:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775496.541237:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775496.541244:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775775496.541252:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775496.541265:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099633536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775496.541283:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775496.541290:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775496.541300:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099633536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775496.541334:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775496.541342:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775496.541350:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775496.541353:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775496.541354:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775496.541357:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775496.541365:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17211 00000400:00000200:1.0:1775775496.541380:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775496.541393:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775496.541401:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775496.541413:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541424:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.541434:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541444:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775496.541453:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775496.541456:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775496.541463:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775496.541470:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775496.541476:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775496.541488:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775496.541498:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541508:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775496.541515:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541523:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775496.541529:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775496.541538:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775496.541546:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775496.541553:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775496.541560:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775496.541567:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.541573:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775496.541580:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775496.541588:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.541597:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775496.541606:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775775496.541614:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775496.541624:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775496.541629:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775496.541638:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775496.541642:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775496.541648:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775496.541656:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775775496.541663:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775496.541667:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775496.541672:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00100000:1.0:1775775496.541675:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099633664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775496.541698:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775496.541705:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775496.541721:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099633664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775496.541772:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775496.541811:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17221) 00000800:00000200:2.1:1775775496.542037:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775496.542039:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775496.542229:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775496.542231:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775496.542501:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775496.542503:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775497.308017:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775497.308020:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775497.308024:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775497.819855:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311885558, at 4311885568 00000100:00000010:1.0:1775775497.819870:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775775499.355983:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775499.355987:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775499.355989:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775500.379994:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775500.379997:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775500.380000:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775501.660493:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775501.660500:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775501.661093:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775501.661095:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775501.661339:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775501.661341:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775501.661583:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775501.661585:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775501.661759:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775501.661761:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000020:00000001:1.0:1775775502.427958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775502.427961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775502.427964:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775502.939872:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311890677, at 4311890688 00000100:00000010:1.0:1775775502.939888:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000020:00000001:1.0:1775775503.452046:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775503.452049:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775503.452052:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775505.500054:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775505.500060:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775505.500066:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775506.524002:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775506.524007:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775506.524011:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775506.780559:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775506.780567:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775506.781118:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775506.781121:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775506.781476:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775506.781479:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775506.781806:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775506.781809:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775506.782119:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775506.782122:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000100:00000040:1.0:1775775508.059900:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311895797, at 4311895808 00000100:00000010:1.0:1775775508.059914:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775508.571966:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775508.571973:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775508.571977:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775509.596010:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775509.596014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775509.596016:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775511.643997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775511.644001:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775511.644003:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775511.900415:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775511.900892:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775511.901194:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775511.901355:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775511.901518:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775512.667990:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775512.667993:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775512.667995:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775513.179868:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311900917, at 4311900928 00000100:00000010:1.0:1775775513.179879:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775514.716023:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775514.716027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775514.716031:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775515.740077:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775515.740080:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775515.740083:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775517.020716:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775517.020725:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775517.021534:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775517.021537:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775517.021883:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775517.021886:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775517.022245:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775517.022248:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775517.022542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775517.022544:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775517.788003:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775517.788007:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775517.788011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775518.299915:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311906038, at 4311906048 00000100:00000010:1.0:1775775518.299947:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775518.811968:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775518.811973:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775518.811978:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775520.860051:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775520.860054:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775520.860057:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775521.883999:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775521.884003:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775521.884008:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775522.139918:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775522.139932:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775522.139935:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775522.139940:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775522.139946:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775522.139955:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17237 00000400:00000200:1.0:1775775522.139968:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775522.139976:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775522.139979:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775522.139982:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.139985:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.139987:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.139990:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775522.139994:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775522.139995:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775522.139998:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775522.140002:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775522.140003:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775522.140005:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775522.140007:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140009:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140010:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140011:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775522.140013:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775522.140016:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140017:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775522.140020:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775522.140023:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775522.140025:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140027:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775522.140029:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775522.140041:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775522.140044:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775522.140047:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775522.140048:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140051:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775522.140054:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775522.140059:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775522.140068:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775775522.140075:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775522.140084:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099636352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775522.140094:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775522.140098:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775522.140102:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099636352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775522.140124:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775522.140127:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775522.140130:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775522.140132:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775522.140133:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775522.140135:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775522.140137:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17237 00000400:00000200:1.0:1775775522.140139:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775522.140141:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775522.140143:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775522.140145:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140147:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140148:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140150:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775522.140151:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775522.140152:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775522.140153:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775522.140154:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775522.140155:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775522.140157:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775522.140159:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140160:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140161:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140162:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775522.140164:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775522.140166:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140168:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775522.140169:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775522.140174:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775522.140175:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140177:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775522.140181:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775522.140183:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140185:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775522.140188:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775522.140190:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775522.140192:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775522.140193:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140195:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775522.140196:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775522.140197:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775522.140200:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775775522.140203:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775522.140208:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099636480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775522.140218:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775522.140221:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775522.140226:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099636480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775522.140254:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775522.140258:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775522.140263:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775522.140264:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775522.140267:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775522.140271:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775522.140282:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17237 00000400:00000200:1.0:1775775522.140288:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775522.140292:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775522.140295:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775522.140297:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140299:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140304:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140308:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775522.140310:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775522.140313:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775522.140316:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775522.140318:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775522.140323:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775522.140325:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775522.140327:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140329:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140333:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140334:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775522.140336:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775522.140337:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140339:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775522.140344:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775522.140347:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775522.140352:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140354:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775522.140358:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775522.140364:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140368:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775522.140375:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775522.140379:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775522.140383:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775522.140384:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140388:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775522.140390:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775522.140392:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775522.140397:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775522.140401:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775522.140407:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099636608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775522.140430:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775522.140432:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775522.140439:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099636608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775522.140451:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775522.140455:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775522.140461:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775522.140464:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775522.140467:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775522.140471:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775522.140474:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17237 00000400:00000200:1.0:1775775522.140489:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775522.140499:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775522.140501:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775522.140503:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140505:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140509:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140514:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775522.140517:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775522.140520:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775522.140522:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.1:1775775522.140524:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:1.0:1775775522.140526:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000800:00000200:2.1:1775775522.140530:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775522.140531:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775522.140540:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775522.140547:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140553:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140561:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140565:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775522.140571:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775522.140577:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140583:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775522.140587:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775522.140592:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775522.140595:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140599:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775522.140603:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775522.140609:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140613:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775522.140618:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775522.140622:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775522.140625:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775522.140626:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140628:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775522.140630:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775522.140633:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775522.140638:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775522.140642:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775522.140650:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099636736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775522.140659:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775522.140661:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775522.140665:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099636736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775522.140679:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775522.140682:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775522.140686:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775522.140689:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775522.140691:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775522.140696:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775522.140702:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17237 00000400:00000200:1.0:1775775522.140711:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775522.140714:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775522.140717:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775522.140721:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140724:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140728:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140731:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775522.140735:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775522.140738:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775522.140741:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775522.140743:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775522.140747:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775522.140751:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775522.140757:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140762:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775522.140766:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140771:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775522.140776:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775522.140804:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775522.140806:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775522.140808:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775522.140810:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775522.140813:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140816:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775522.140819:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775522.140823:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140826:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775522.140831:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775775522.140835:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775522.140838:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775522.140842:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775522.140844:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775522.140846:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775522.140849:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775522.140854:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775522.140858:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775522.140884:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099636864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775522.140900:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775522.140904:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775522.140911:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099636864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775522.140944:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775522.140952:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17247) 00000800:00000200:2.1:1775775522.141095:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775522.141096:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775522.141281:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775522.141283:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775522.141456:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775522.141457:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775522.141629:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775522.141630:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1775775523.419906:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311911157, at 4311911168 00000100:00000010:1.0:1775775523.419926:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775775523.931997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775523.932001:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775523.932004:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775524.956042:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775524.956045:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775524.956048:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775527.004052:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775527.004057:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775527.004061:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775527.260716:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775527.260728:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775527.261408:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775527.261410:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775527.261748:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775527.261750:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775527.261972:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775527.261974:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775527.262154:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775527.262156:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775775528.027997:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775528.028001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775528.028004:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775528.539939:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311916278, at 4311916288 00000100:00000010:1.0:1775775528.539960:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775530.076023:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775530.076027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775530.076031:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775531.100058:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775531.100061:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775531.100064:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775532.380632:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775532.380644:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775532.381532:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775532.381535:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775532.381906:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775532.381909:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775532.382212:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775532.382214:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775532.382511:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775532.382514:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775775533.148014:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775533.148018:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775533.148020:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775533.659919:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311921398, at 4311921408 00000100:00000010:1.0:1775775533.659934:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775534.172004:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775534.172009:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775534.172013:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775536.220065:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775536.220069:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775536.220072:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775537.243960:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775537.243966:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775537.243969:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775537.500430:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775537.500435:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775537.501037:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775537.501042:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775537.501447:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775537.501448:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775537.501680:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775537.501682:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775537.501872:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775537.501874:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000100:00000040:1.0:1775775538.779887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311926517, at 4311926528 00000100:00000010:1.0:1775775538.779906:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775775539.291931:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775539.291935:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775539.291939:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775540.315992:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775540.315997:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775540.316001:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775542.364005:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775542.364011:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775542.364014:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775542.620489:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775542.620497:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775542.621026:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775542.621028:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775542.621226:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775542.621228:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775542.621440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775542.621442:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775542.621670:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775542.621672:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775775543.387963:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775543.387967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775543.387969:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775543.899883:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311931637, at 4311931648 00000100:00000010:1.0:1775775543.899897:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775775545.435974:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775545.435978:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775545.435982:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775546.460088:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775546.460092:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775546.460094:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775547.740297:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775547.740301:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775547.740702:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775547.740704:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775547.740929:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775547.740930:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775547.741104:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775547.741106:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775547.741340:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775547.741342:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775775548.507982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775548.507985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775548.507988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775549.019895:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311936757, at 4311936768 00000100:00000010:1.0:1775775549.019910:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775549.531950:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775549.531953:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775549.531956:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775551.580043:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775551.580046:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775551.580049:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775552.603979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775552.603982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775552.603985:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775552.860384:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775552.860388:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775552.861066:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775552.861068:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775552.861265:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775552.861266:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775552.861482:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775552.861484:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775552.861751:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775552.861752:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1775775554.139857:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311941877, at 4311941888 00000100:00000010:1.0:1775775554.139877:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775554.651972:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775554.651976:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775554.651980:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775555.675995:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775555.676000:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775555.676003:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775557.723948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775557.723953:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775557.723956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775557.980471:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775557.980477:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775557.980982:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775557.980984:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775557.981165:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775557.981168:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775557.981417:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775557.981418:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775557.981627:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775557.981628:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775558.747997:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775558.748000:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775558.748003:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775559.259903:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311946997, at 4311947008 00000100:00000010:1.0:1775775559.259922:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775775560.795998:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775560.796001:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775560.796004:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775561.820012:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775561.820016:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775561.820020:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775562.780016:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775562.780029:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775562.780032:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775562.780041:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775562.780048:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775562.780054:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775562.780058:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775562.780061:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775562.780065:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775563.099921:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775563.099931:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775563.099934:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775563.099939:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775563.099943:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775563.099951:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17278 00000400:00000200:1.0:1775775563.099962:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775563.099969:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775563.099972:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775563.099974:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.099978:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.099980:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.099982:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775563.099987:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775563.099988:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775563.099991:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775563.099993:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775563.099995:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775563.099997:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775563.100000:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100001:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100002:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100004:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775563.100006:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775563.100009:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100010:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775563.100013:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775563.100015:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775563.100017:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100019:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775563.100021:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775563.100032:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775775563.100035:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775563.100039:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775563.100040:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100043:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775563.100046:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775563.100051:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775563.100060:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775563.100066:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775563.100075:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099641472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775563.100085:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775563.100090:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775563.100093:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099641472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775563.100114:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775563.100116:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775563.100119:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775563.100120:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775563.100122:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775563.100124:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775563.100126:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17278 00000400:00000200:1.0:1775775563.100129:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775563.100130:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775563.100132:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775563.100134:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100136:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100137:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100138:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775563.100141:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775563.100141:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775563.100142:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775563.100144:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775563.100145:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775563.100161:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775563.100163:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100165:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100166:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100167:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775563.100169:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775563.100171:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100173:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775563.100175:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775563.100181:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775563.100183:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100185:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775563.100191:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775563.100195:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100198:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775563.100202:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775775563.100206:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775563.100211:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775563.100212:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100216:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775563.100218:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775563.100220:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775563.100224:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775563.100230:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775563.100235:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099641600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775563.100248:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775563.100251:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775563.100256:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099641600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775563.100267:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775563.100269:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775563.100274:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775563.100277:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775563.100279:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775563.100283:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775563.100285:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17278 00000400:00000200:1.0:1775775563.100288:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775563.100290:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775563.100294:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775563.100296:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100299:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100301:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100303:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775563.100308:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775563.100311:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775563.100313:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775563.100315:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775563.100317:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775563.100321:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775563.100323:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100324:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100325:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100327:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775563.100328:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775563.100333:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100334:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775563.100335:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775563.100336:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775563.100338:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100339:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775563.100341:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775563.100345:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100346:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775563.100350:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775775563.100353:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775563.100355:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775563.100357:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100361:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775563.100363:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775563.100365:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775563.100371:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775563.100375:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775563.100381:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099641728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775563.100391:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775563.100394:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775563.100399:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099641728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775563.100407:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775563.100409:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775563.100414:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775563.100416:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775563.100417:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775563.100419:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775563.100425:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17278 00000400:00000200:1.0:1775775563.100429:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775563.100433:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775563.100435:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775563.100440:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100442:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100446:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100451:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775563.100454:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775563.100456:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775563.100458:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775563.100461:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775563.100464:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775563.100468:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775563.100472:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100474:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100475:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100477:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775563.100480:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775563.100485:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100488:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775563.100493:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775563.100496:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775563.100498:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100501:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775563.100503:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775563.100505:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100509:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775563.100511:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775775563.100513:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775563.100516:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775563.100518:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100520:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775563.100521:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775563.100523:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775563.100528:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000800:00000200:2.1:1775775563.100529:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775563.100531:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775563.100533:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00100000:1.0:1775775563.100537:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099641856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775563.100550:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775563.100552:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775563.100555:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099641856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775563.100566:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775563.100571:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775563.100573:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775563.100577:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775563.100578:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775563.100582:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775563.100585:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17278 00000400:00000200:1.0:1775775563.100592:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775563.100597:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775563.100602:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775563.100608:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100610:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100613:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100616:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775563.100620:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775563.100622:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775563.100625:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775563.100628:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775563.100633:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775563.100635:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775563.100641:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100643:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775563.100653:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100657:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775563.100658:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775563.100660:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775563.100665:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775563.100666:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775563.100668:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775563.100669:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100670:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775563.100672:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775563.100676:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100677:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775563.100680:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775775563.100682:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775563.100684:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775563.100685:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775563.100690:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775563.100692:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775563.100694:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775563.100698:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775775563.100705:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775563.100711:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099641984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775563.100719:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775563.100723:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775563.100729:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099641984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775563.100736:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775563.100743:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17288) 00000800:00000200:2.1:1775775563.100973:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775563.100974:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775563.101214:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775563.101216:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775563.101399:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775563.101401:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775563.101574:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775563.101575:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1775775563.804048:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775563.804058:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775563.804061:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775563.804069:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775563.804074:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775563.804080:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775563.804084:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775563.804087:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775563.804091:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775563.868003:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775563.868008:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775563.868012:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775564.379894:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311952117, at 4311952128 00000100:00000010:1.0:1775775564.379907:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398ba20. 00000100:00000001:1.0:1775775564.827853:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775564.827859:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775564.827860:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775564.827864:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775564.827867:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775564.827870:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775564.827872:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775564.827874:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775564.827876:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775564.892024:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775564.892033:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775564.892037:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775565.851848:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775565.851854:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775565.851855:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775565.851859:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775565.851862:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775565.851865:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775565.851867:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775565.851869:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775565.851872:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775566.875858:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775566.875864:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775566.875866:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775566.875872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775566.875875:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775566.875879:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775566.875882:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775566.875885:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775566.875894:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775566.939932:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775566.939936:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775566.939940:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775567.899852:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775567.899857:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775567.899858:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775567.899862:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775567.899865:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775567.899868:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775567.899870:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775567.899872:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775567.899873:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775567.963969:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775567.963972:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775567.963975:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775568.219905:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775568.219913:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775568.219916:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775568.219920:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775568.219923:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775568.219930:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17283 00000400:00000200:1.0:1775775568.219939:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775568.219944:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775568.219947:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775568.219950:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.219953:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.219955:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.219957:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775568.219960:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775568.219961:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775568.219964:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775568.219966:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775568.219967:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775568.219969:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775568.219971:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.219973:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.219974:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.219976:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775568.219978:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775568.219981:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775568.219983:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775568.219985:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775568.219987:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775568.219989:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.219991:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775568.219992:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775568.220012:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775775568.220015:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775568.220019:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775568.220020:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220023:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775568.220025:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775568.220031:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775568.220040:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775775568.220047:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775568.220112:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099642112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775568.220124:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775568.220127:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775568.220131:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099642112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775568.220170:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775568.220173:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775568.220176:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775568.220177:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775568.220179:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775568.220181:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775568.220183:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17283 00000400:00000200:1.0:1775775568.220187:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775568.220189:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775568.220191:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775568.220193:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220195:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.220196:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220198:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775568.220200:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775568.220201:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775568.220202:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775568.220204:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775568.220205:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775568.220207:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775568.220210:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220211:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.220213:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220214:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775568.220216:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775568.220218:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220219:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775568.220221:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775568.220225:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775568.220227:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220228:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775568.220232:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775568.220234:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220235:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775568.220240:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775775568.220242:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775568.220244:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775568.220246:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220247:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775568.220248:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775568.220250:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775568.220254:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775775568.220258:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775568.220262:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099642240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775568.220270:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775568.220274:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775568.220281:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099642240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775568.220298:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775568.220304:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775568.220311:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775568.220314:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775568.220318:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775568.220338:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775568.220343:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17283 00000400:00000200:1.0:1775775568.220351:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775568.220358:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775568.220362:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775568.220369:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220373:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.220378:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220380:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775568.220382:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775568.220383:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775568.220386:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775568.220389:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775568.220393:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775568.220409:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775568.220413:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220416:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.220417:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220418:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775568.220420:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775568.220422:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220423:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775568.220425:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775568.220428:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775568.220431:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220432:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775568.220435:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775568.220439:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220440:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775568.220445:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775775568.220448:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775568.220450:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775568.220452:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220453:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775568.220455:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775568.220458:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775568.220462:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775568.220468:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775568.220476:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099642368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775568.220498:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775568.220502:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775568.220507:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099642368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775568.220522:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775568.220524:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775568.220529:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775568.220531:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775568.220533:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775568.220538:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775568.220542:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17283 00000400:00000200:1.0:1775775568.220551:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775568.220572:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775568.220577:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775568.220582:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220585:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.220591:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220597:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775568.220603:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775568.220607:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775568.220611:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775568.220615:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775568.220620:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775568.220626:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775568.220634:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220636:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.220637:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220639:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775568.220642:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775568.220648:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220652:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775568.220671:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775568.220674:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775568.220677:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220680:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775568.220683:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775568.220687:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220688:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775568.220692:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775775568.220698:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775568.220706:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:2.1:1775775568.220708:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775775568.220710:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775568.220712:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775775568.220714:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775568.220716:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775568.220718:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775568.220723:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775568.220728:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775568.220753:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099642496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775568.220781:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775568.220784:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775568.220793:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099642496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775568.220811:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775568.220817:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775568.220825:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775568.220827:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775568.220829:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775568.220832:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775568.220836:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17283 00000400:00000200:1.0:1775775568.220841:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775568.220843:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775568.220845:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775568.220847:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220849:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.220851:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220874:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775568.220878:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775568.220880:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775568.220881:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775568.220883:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775568.220886:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775568.220889:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775568.220894:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220897:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775568.220901:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220906:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775568.220909:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775568.220914:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775568.220919:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775568.220923:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775568.220928:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775568.220930:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220933:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775568.220935:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775568.220940:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220946:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775568.220959:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775775568.220962:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775568.220968:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775568.220970:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775568.220975:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775568.220978:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775568.220980:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775568.220986:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775568.220989:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775568.220999:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099642624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775568.221014:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775568.221018:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775568.221024:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099642624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775568.221053:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775568.221069:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17293) 00000800:00000200:2.1:1775775568.221310:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775568.221311:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775568.221490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775568.221492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775568.221688:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775568.221690:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775568.221903:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775568.221905:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775775568.923940:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775568.923947:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775568.923950:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775568.923955:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775568.923961:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775568.923965:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775568.923968:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775568.923971:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775568.923973:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775569.499875:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311957238, at 4311957248 00000100:00000010:1.0:1775775569.499885:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000001:1.0:1775775569.947856:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775569.947862:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775569.947864:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775569.947869:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775569.947872:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775569.947877:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775569.947879:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775569.947882:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775569.947885:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775570.012080:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775570.012083:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775570.012086:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775570.971889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775570.971895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775570.971897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775570.971901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775570.971904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775570.971906:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775570.971908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775570.971910:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775570.971912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775571.035980:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775571.035984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775571.035986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775571.995903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775571.995909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775571.995910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775571.995915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775571.995918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775571.995922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775571.995924:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775571.995926:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775571.995929:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775573.019890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775573.019895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775573.019897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775573.019901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775573.019904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775573.019907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775573.019909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775573.019911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775573.019913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775573.083950:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775573.083953:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775573.083955:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775573.340484:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775573.340489:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775573.341081:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775573.341083:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775573.341309:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775573.341311:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775573.341531:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775573.341532:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775573.341703:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775573.341705:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1775775574.043852:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775574.043859:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775574.043861:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775574.043868:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775574.043872:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775574.043876:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775574.043878:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775574.043881:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775574.043884:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775574.107987:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775574.107991:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775574.107994:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775574.619882:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311962357, at 4311962368 00000100:00000010:1.0:1775775574.619898:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:1.0:1775775575.067849:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775575.067854:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775575.067855:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775575.067860:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775575.067863:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775575.067866:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775575.067868:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775575.067870:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775575.067872:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775576.091878:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775576.091885:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775576.091887:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775576.091892:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775576.091895:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775576.091899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775576.091901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775576.091903:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775576.091906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775576.156024:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775576.156027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775576.156030:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775577.115886:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775577.115892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775577.115893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775577.115897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775577.115900:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775577.115916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775577.115919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775577.115921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775577.115923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775577.179977:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775577.179982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775577.179985:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775578.139882:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775578.139919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775578.139921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775578.139927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775578.139931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775578.139936:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775578.139938:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775578.139940:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775578.139942:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775578.460400:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775578.460824:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775578.461071:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775578.461238:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775578.461379:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775579.163887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775579.163894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775579.163896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775579.163900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775579.163902:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775579.163906:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775579.163908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775579.163910:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775579.163912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775579.227977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775579.227980:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775579.227983:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775579.739886:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311967477, at 4311967488 00000100:00000010:1.0:1775775579.739902:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000001:1.0:1775775580.187880:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775580.187887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775580.187888:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775580.187893:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775580.187896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775580.187899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775580.187901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775580.187903:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775580.187904:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775580.252010:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775580.252014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775580.252017:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775581.211931:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775581.211939:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775581.211941:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775581.211947:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775581.211951:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775581.211955:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775581.211958:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775581.211961:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775581.211963:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775582.235897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775582.235905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775582.235907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775582.235913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775582.235917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775582.235925:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775582.235928:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775582.235931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775582.235934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775582.299990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775582.299994:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775582.299997:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775583.259891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775583.259897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775583.259898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775583.259903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775583.259906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775583.259909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775583.259911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775583.259913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775583.259915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775583.323981:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775583.323984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775583.323986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775583.579899:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775583.579910:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775583.579913:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775583.579917:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775583.579922:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775583.579930:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17298 00000400:00000200:1.0:1775775583.579940:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775583.579946:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775583.579950:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775583.579953:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.579956:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.579959:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.579961:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775583.579965:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775583.579967:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775583.579970:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775583.579972:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775583.579976:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775583.579979:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775583.579982:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.579984:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.579986:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.579988:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775583.579991:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775583.579995:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775583.579998:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775583.580001:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775583.580005:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775583.580008:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580010:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775583.580013:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775583.580019:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775775583.580023:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775583.580028:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775583.580030:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580033:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775583.580035:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775583.580040:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775583.580048:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775775583.580055:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775583.580063:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099644032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775583.580071:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775583.580075:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775583.580079:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099644032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775583.580100:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775583.580103:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775583.580108:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775583.580109:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775583.580111:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775583.580114:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775583.580116:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17298 00000400:00000200:1.0:1775775583.580121:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775583.580125:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775583.580128:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775583.580133:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580136:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.580139:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580142:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775583.580146:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775583.580150:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775583.580153:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775583.580156:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775583.580160:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775583.580165:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775583.580175:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580180:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.580184:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580189:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775583.580194:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775583.580197:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580200:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775583.580203:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775583.580210:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775583.580216:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580221:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775583.580228:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775583.580235:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580240:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775583.580247:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775583.580252:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775583.580257:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775583.580260:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580263:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775583.580265:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775583.580268:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775583.580276:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775583.580281:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775583.580290:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099644160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775583.580309:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775583.580313:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775583.580323:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099644160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775583.580341:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775583.580345:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775583.580350:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775583.580354:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775583.580357:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775583.580361:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775583.580367:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17298 00000400:00000200:1.0:1775775583.580377:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775583.580383:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775583.580388:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775583.580404:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580409:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.580416:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580422:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775583.580427:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775583.580430:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775583.580435:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775583.580439:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775583.580444:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775583.580448:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775583.580453:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580458:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.580465:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580470:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775583.580475:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775583.580483:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580487:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775583.580490:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775583.580494:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775583.580498:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580500:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775583.580505:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775583.580518:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775583.580523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775583.580524:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775583.580530:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000100:00000010:1.0:1775775583.580532:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775583.580536:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775583.580538:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775583.580540:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580541:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775583.580543:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775583.580546:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775583.580551:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775583.580557:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775583.580565:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099644288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775583.580589:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775583.580594:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775583.580601:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099644288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775583.580628:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775583.580633:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775583.580640:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775583.580643:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775583.580647:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775583.580654:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775583.580661:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17298 00000400:00000200:1.0:1775775583.580667:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775583.580676:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775583.580681:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775583.580688:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580692:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.580696:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580699:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775583.580701:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775583.580704:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775583.580708:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775583.580710:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775583.580714:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775583.580718:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775583.580724:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580728:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.580732:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580737:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775583.580742:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775583.580749:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775583.580754:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775583.580759:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775583.580764:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775583.580768:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580770:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775583.580774:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775583.580840:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580844:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775583.580851:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775583.580856:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775583.580861:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775583.580864:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.580872:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775583.580876:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775583.580879:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775583.580886:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775583.580891:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775583.580903:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099644416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775583.580918:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775583.580924:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775583.580935:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099644416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775583.580952:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775583.580958:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775583.580966:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775583.580970:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775583.580972:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775583.580977:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775583.580985:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17298 00000400:00000200:1.0:1775775583.580994:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775583.581002:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775583.581008:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775583.581016:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.581021:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.581026:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.581031:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775583.581036:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775583.581039:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775583.581043:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775583.581051:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775583.581057:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775583.581062:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775583.581069:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.581073:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775583.581077:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775583.581082:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775583.581087:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775583.581094:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775583.581099:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775583.581103:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775583.581109:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775583.581114:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.581119:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775583.581125:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775583.581131:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.581136:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775583.581144:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775583.581151:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775583.581158:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775583.581159:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775583.581161:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775583.581164:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775583.581168:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775583.581175:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775583.581181:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775583.581192:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099644544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775583.581212:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775583.581216:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775583.581226:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099644544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775583.581244:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775583.581251:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17309) 00000800:00000200:2.1:1775775583.581340:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775583.581343:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1775775583.581663:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775583.581666:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1775775583.581995:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775583.581997:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1775775583.582380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775583.582382:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 57 secs later) 00000100:00000001:1.0:1775775584.283874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775584.283880:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775584.283882:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775584.283886:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775584.283889:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775584.283892:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775584.283894:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775584.283896:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775584.283898:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775584.859877:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311972597, at 4311972608 00000100:00000010:1.0:1775775584.859895:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000001:1.0:1775775585.307875:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775585.307881:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775585.307882:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775585.307887:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775585.307890:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775585.307893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775585.307895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775585.307897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775585.307899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775585.371983:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775585.371987:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775585.371990:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775586.331844:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775586.331852:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775586.331854:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775586.331860:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775586.331864:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775586.331868:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775586.331871:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775586.331873:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775586.331877:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775586.395984:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775586.395987:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775586.395989:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775587.355901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775587.355907:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775587.355909:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775587.355914:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775587.355918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775587.355922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775587.355925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775587.355928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775587.355931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775588.379852:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775588.379860:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775588.379862:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775588.379874:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775588.379878:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775588.379882:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775588.379884:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775588.379887:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775588.379890:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775588.444016:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775588.444021:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775588.444025:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775588.700685:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775588.700691:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775775588.701770:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775588.701772:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775775588.701990:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775588.701991:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775775588.702246:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775588.702248:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775775588.702467:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775588.702469:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000100:00000001:1.0:1775775589.403922:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775589.403931:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775589.403933:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775589.403941:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775589.403946:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775589.403950:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775589.403954:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775589.403958:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775589.403961:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775589.468064:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775589.468070:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775589.468074:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775589.979933:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311977718, at 4311977728 00000100:00000010:1.0:1775775589.979962:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:1.0:1775775590.427883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775590.427892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775590.427895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775590.427902:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775590.427908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775590.427913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775590.427916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775590.427920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775590.427923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775591.451914:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775591.451924:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775591.451926:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775591.451934:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775591.451939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775591.451944:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775591.451948:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775591.451952:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775591.451955:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775591.516027:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775591.516032:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775591.516037:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775592.475918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775592.475927:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775592.475929:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775592.475935:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775592.475939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775592.475944:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775592.475948:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775592.475951:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775592.475953:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775592.540043:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775592.540047:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775592.540050:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775593.500035:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775593.500059:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775593.500063:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775593.500079:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775593.500089:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775593.500099:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775593.500104:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775593.500109:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775593.500116:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775775593.820461:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775593.820467:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775593.821073:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775593.821076:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775593.821271:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775593.821273:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775593.821473:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775593.821474:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775593.821684:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775593.821686:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000100:00000001:1.0:1775775594.523878:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775594.523884:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775594.523885:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775594.523889:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775594.523892:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775594.523895:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775594.523897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775594.523899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775594.523900:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775594.587963:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775594.587968:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775594.587972:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775595.099852:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311982837, at 4311982848 00000100:00000010:1.0:1775775595.099866:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344a80. 00000100:00000001:1.0:1775775595.547855:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775595.547861:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775595.547863:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775595.547868:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775595.547871:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775595.547874:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775595.547876:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775595.547879:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775595.547881:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775595.611964:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775595.611967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775595.611969:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775596.571862:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775596.571867:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775596.571868:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775596.571873:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775596.571876:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775596.571879:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775596.571881:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775596.571883:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775596.571885:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775597.595896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775597.595902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775597.595904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775597.595908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775597.595911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775597.595915:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775597.595917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775597.595919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775597.595921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775597.659996:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775597.660000:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775597.660003:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775598.619882:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775598.619887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775598.619889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775598.619893:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775598.619896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775598.619899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775598.619901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775598.619903:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775598.619905:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775598.683937:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775598.683946:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775598.683952:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775598.940511:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775598.940516:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775598.941246:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775598.941248:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775598.941465:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775598.941467:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775598.941695:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775598.941697:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775598.942043:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775598.942047:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000100:00000001:1.0:1775775599.643924:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775599.643934:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775599.643936:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775599.643944:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775599.643949:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775599.643954:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775599.643958:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775599.643961:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775599.643965:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775600.219941:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311987957, at 4311987968 00000100:00000010:1.0:1775775600.219963:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000100:00000001:1.0:1775775600.667922:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775600.667932:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775600.667935:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775600.667943:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775600.667948:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775600.667953:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775600.667957:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775600.667961:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775600.667964:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775600.732033:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775600.732036:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775600.732040:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775601.691894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775601.691901:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775601.691902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775601.691909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775601.691914:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775601.691919:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775601.691922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775601.691926:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775601.691929:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775601.756007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775601.756011:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775601.756014:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775602.715898:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775602.715906:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775602.715908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775602.715914:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775602.715918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775602.715923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775602.715925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775602.715928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775602.715931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775603.739901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775603.739910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775603.739912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775603.739920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775603.739925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775603.739930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775603.739933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775603.739937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775603.739940:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775603.803958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775603.803963:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775603.803967:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775604.060595:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775604.060602:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775604.061083:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775604.061085:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775604.061364:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775604.061367:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775604.061663:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775604.061666:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775604.061962:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775604.061965:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000001:1.0:1775775604.763882:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775604.763888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775604.763890:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775604.763895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775604.763899:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775604.763903:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775604.763906:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775604.763908:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775604.763911:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775604.827937:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775604.827940:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775604.827943:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775605.339889:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311993077, at 4311993088 00000100:00000010:1.0:1775775605.339907:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000100:00000001:1.0:1775775605.787871:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775605.787877:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775605.787879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775605.787885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775605.787889:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775605.787893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775605.787896:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775605.787899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775605.787901:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775606.811877:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775606.811883:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775606.811885:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775606.811890:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775606.811893:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775606.811896:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775606.811898:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775606.811900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775606.811902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775606.875952:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775606.875958:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775606.875962:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775607.835899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775607.835907:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775607.835908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775607.835914:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775607.835919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775607.835922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775607.835925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775607.835928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775607.835930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775607.899980:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775607.899984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775607.899988:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775608.859855:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775608.859861:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775608.859863:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775608.859867:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775608.859870:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775608.859873:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775608.859875:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775608.859877:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775608.859879:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775609.180749:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775609.180757:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775609.182012:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775609.182016:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775609.182372:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775609.182375:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775609.182846:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775609.182850:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775609.183217:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775609.183220:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:1.0:1775775609.883927:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775609.883951:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775609.883953:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775609.883959:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775609.883963:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775609.883967:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775609.883970:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775609.883974:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775609.883978:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775609.948067:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775609.948098:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775609.948102:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775610.459888:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4311998198, at 4311998208 00000100:00000010:1.0:1775775610.459908:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000001:1.0:1775775610.907888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775610.907897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775610.907900:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775610.907907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775610.907912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775610.907917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775610.907921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775610.907924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775610.907927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775611.931870:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775611.931877:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775611.931879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775611.931885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775611.931889:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775611.931893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775611.931896:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775611.931899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775611.931902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775612.955883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775612.955890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775612.955892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775612.955897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775612.955901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775612.955905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775612.955907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775612.955910:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775612.955912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775613.020002:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775613.020005:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775613.020008:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775613.979969:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775613.979978:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775613.979980:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775613.979987:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775613.979992:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775613.979997:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775613.980000:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775613.980004:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775613.980006:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775614.300853:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775614.300862:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775614.301936:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775614.301939:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775614.302179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775614.302181:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775614.302412:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775614.302414:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775614.302625:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775614.302627:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:1.0:1775775615.003841:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775615.003846:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775615.003848:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775615.003853:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775615.003856:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775615.003859:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775615.003861:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775615.003864:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775615.003866:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775615.579915:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312003318, at 4312003328 00000100:00000010:1.0:1775775615.579930:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000001:1.0:1775775616.027850:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775616.027855:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775616.027856:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775616.027861:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775616.027864:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775616.027867:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775616.027870:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775616.027872:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775616.027874:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775616.092093:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775616.092098:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775616.092101:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775617.051867:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775617.051888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775617.051890:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775617.051894:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775617.051897:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775617.051900:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775617.051902:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775617.051904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775617.051906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775618.075948:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775618.075959:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775618.075962:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775618.075986:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775618.075991:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775618.075996:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775618.076001:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775618.076005:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775618.076009:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775619.099977:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775619.099986:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775619.099989:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775619.099996:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775619.100001:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775619.100005:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775619.100009:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775619.100012:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775619.100015:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775619.163976:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775619.163981:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775619.163984:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775619.420362:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775619.420372:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775619.421071:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775619.421072:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775619.421326:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775619.421329:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775619.421583:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775619.421585:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775619.421874:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775619.421876:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1775775620.123903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775620.123911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775620.123913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775620.123920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775620.123924:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775620.123928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775620.123931:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775620.123934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775620.123937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775620.699944:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312008437, at 4312008448 00000100:00000010:1.0:1775775620.699969:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000001:1.0:1775775621.147843:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775621.147849:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775621.147850:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775621.147854:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775621.147857:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775621.147860:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775621.147862:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775621.147864:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775621.147866:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775622.171862:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775622.171868:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775622.171869:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775622.171874:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775622.171878:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775622.171882:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775622.171884:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775622.171886:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775622.171889:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775622.235958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775622.235961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775622.235964:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775623.195834:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775623.195839:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775623.195840:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775623.195845:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775623.195848:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775623.195858:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775623.195860:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775623.195862:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775623.195864:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775624.219875:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775624.219886:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775624.219889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775624.219893:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775624.219896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775624.219899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775624.219901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775624.219904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775624.219905:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775775624.540458:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775624.540460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775624.541186:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775624.541188:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775624.541416:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775624.541419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775624.541624:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775624.541626:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775624.541904:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775624.541906:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1775775625.244745:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775625.244760:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775625.244764:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775625.244774:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775625.244830:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775625.244839:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775625.244846:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775625.244851:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775625.244857:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775625.307999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775625.308005:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775625.308010:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775625.819979:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312013557, at 4312013568 00000100:00000010:1.0:1775775625.820001:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000020:00000001:1.0:1775775628.379965:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775628.379970:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775628.379974:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775629.659924:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775629.659935:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775629.659938:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775629.659942:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775629.659947:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775629.659954:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17344 00000400:00000200:1.0:1775775629.659964:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775629.659972:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775629.660019:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775629.660023:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660026:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.660029:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660031:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775629.660035:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775629.660037:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775629.660040:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775629.660042:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775629.660045:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775629.660047:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775629.660049:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660051:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.660053:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660055:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775629.660057:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775629.660061:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660063:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775629.660066:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775629.660069:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775629.660071:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660074:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775629.660076:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775629.660084:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775775629.660088:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775629.660092:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775629.660093:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660097:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775629.660100:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775629.660105:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775629.660116:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775775629.660123:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775629.660132:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099649792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775629.660148:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775629.660152:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775629.660157:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099649792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775629.660178:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775629.660182:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775629.660185:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775629.660187:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775629.660188:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775629.660191:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775629.660193:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17344 00000400:00000200:1.0:1775775629.660197:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775629.660200:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775629.660203:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775629.660207:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660209:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.660211:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660213:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775629.660216:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775629.660217:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775629.660219:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775629.660221:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775629.660223:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775629.660225:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775629.660230:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660232:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.660235:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660240:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775629.660244:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775629.660251:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660257:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775629.660261:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775629.660278:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775629.660282:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660286:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775629.660293:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775629.660299:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660305:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775629.660312:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775775629.660316:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775629.660323:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775629.660326:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660342:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775629.660345:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775629.660348:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775629.660354:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775775629.660360:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775629.660373:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099649920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775629.660406:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775629.660410:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775629.660417:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099649920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775629.660435:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775629.660439:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775629.660445:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775629.660447:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775629.660451:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775629.660456:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775629.660463:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17344 00000400:00000200:1.0:1775775629.660474:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775629.660481:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775629.660484:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775629.660488:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660492:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.660498:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660514:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775629.660518:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775629.660520:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775629.660523:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775629.660525:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775629.660529:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775629.660534:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775629.660540:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660544:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.660549:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660553:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000800:00000200:3.1:1775775629.660556:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:1.0:1775775629.660557:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000800:00000200:3.1:1775775629.660560:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000100:00080000:1.0:1775775629.660564:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660570:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775629.660574:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775629.660578:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775629.660583:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660589:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775629.660593:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775629.660598:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660601:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775629.660605:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775775629.660610:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775629.660615:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775629.660618:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660621:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775629.660625:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775629.660628:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775629.660635:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775775629.660643:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775629.660651:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099650048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775629.660667:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775629.660670:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775629.660675:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099650048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775629.660690:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775629.660695:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775629.660701:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775629.660704:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775629.660706:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775629.660710:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775629.660715:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17344 00000400:00000200:1.0:1775775629.660724:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775629.660729:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775629.660732:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775629.660736:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660740:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.660743:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660745:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775629.660748:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775629.660749:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775629.660750:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775629.660752:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775629.660757:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775629.660761:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775629.660768:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660771:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.660818:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660823:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775629.660828:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775629.660835:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775629.660839:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775629.660843:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775629.660848:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775629.660854:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660858:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775629.660861:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775629.660867:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660871:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775629.660877:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775775629.660883:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775629.660888:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775629.660891:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.660896:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775629.660900:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775629.660904:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775629.660909:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775629.660915:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775629.660924:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099650176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775629.660944:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775629.660949:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775629.660957:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099650176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775629.660973:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775629.660978:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775629.660999:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775629.661000:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775629.661002:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775629.661008:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775629.661014:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17344 00000400:00000200:1.0:1775775629.661020:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775629.661025:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775629.661028:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775629.661034:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.661039:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.661044:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.661047:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775629.661052:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775629.661053:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775629.661057:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775629.661060:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775629.661062:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775629.661066:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775629.661071:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.661075:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775629.661078:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775629.661082:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775629.661087:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775629.661093:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775629.661095:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775629.661098:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775629.661103:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775629.661107:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.661112:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775629.661116:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775629.661121:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.661125:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775629.661129:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775775629.661134:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775629.661140:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775629.661142:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775629.661145:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775629.661149:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775629.661151:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775629.661157:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775629.661164:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775629.661174:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099650304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775629.661186:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775629.661190:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775629.661198:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099650304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775629.661214:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775629.661221:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17355) 00000800:00000200:3.1:1775775629.661290:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775629.661291:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775775629.661533:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775629.661534:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775775629.661719:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775629.661720:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775775629.662022:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775629.662024:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000100:00000040:1.0:1775775630.939879:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312018677, at 4312018688 00000100:00000010:1.0:1775775630.939894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000020:00000001:1.0:1775775631.451976:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775631.451979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775631.451982:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775634.524009:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775634.524013:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775634.524017:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775634.780568:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775634.780576:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775634.781207:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775634.781210:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775634.781522:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775634.781525:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775634.781874:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775634.781877:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775775634.782181:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775634.782183:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000100:00000040:1.0:1775775636.059928:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312023798, at 4312023808 00000100:00000010:1.0:1775775636.059938:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775637.595980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775637.595984:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775637.595987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775639.900671:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775639.900676:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775639.901392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775639.901395:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775639.901706:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775639.901709:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775639.902111:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775639.902114:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775639.902461:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775639.902464:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000020:00000001:1.0:1775775640.667986:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775640.667991:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775640.667996:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775641.179967:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312028918, at 4312028928 00000100:00000010:1.0:1775775641.179990:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775643.740067:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775643.740070:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775643.740073:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775645.020622:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775645.021259:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775645.021687:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775645.021915:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775645.022065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775775646.299889:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312034037, at 4312034048 00000100:00000010:1.0:1775775646.299905:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775775646.811999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775646.812003:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775646.812005:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775649.884034:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775649.884038:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775649.884042:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775650.140696:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775650.140703:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775650.141334:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775650.141336:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775650.141644:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775650.141647:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775650.141927:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775650.141930:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775650.142144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775650.142146:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000040:1.0:1775775651.419848:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312039158, at 4312039168 00000100:00000010:1.0:1775775651.419861:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775652.955936:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775652.955939:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775652.955942:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775655.260427:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775655.260434:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775655.261106:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775655.261108:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775655.261408:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775655.261410:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775655.261725:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775655.261728:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775655.261963:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775655.261965:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775656.027970:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775656.027973:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775656.027976:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775656.539883:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312044277, at 4312044288 00000100:00000010:1.0:1775775656.539899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775659.099984:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775659.099988:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775659.099991:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775660.380550:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775660.380554:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775660.381090:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775660.381092:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775660.381275:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775660.381289:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775660.381485:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775660.381487:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775660.381695:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775660.381697:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000040:1.0:1775775661.659888:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312049397, at 4312049408 00000100:00000010:1.0:1775775661.659901:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775662.171964:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775662.171967:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775662.171970:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775665.243982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775665.243986:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775665.243990:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775665.500516:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775665.500522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775665.501153:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775665.501155:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775665.501435:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775665.501436:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775665.501681:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775665.501683:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775665.501908:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775665.501910:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000040:1.0:1775775666.779880:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312054517, at 4312054528 00000100:00000010:1.0:1775775666.779894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775775668.315950:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775668.315954:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775668.315957:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775670.620536:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775670.620543:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775670.621050:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775670.621053:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775670.621338:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775670.621340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775670.621596:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775670.621598:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775670.621870:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775670.621885:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775775671.387923:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775671.387927:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775671.387930:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775671.899923:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312059637, at 4312059648 00000100:00000010:1.0:1775775671.899937:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775674.459979:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775674.459983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775674.459986:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775675.739917:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775675.739926:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775675.739929:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775675.739932:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775675.739936:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775675.739941:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17391 00000400:00000200:1.0:1775775675.739949:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775675.739955:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775675.739958:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775675.739962:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.739965:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.739973:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.739976:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775675.739980:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775675.739982:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775675.739985:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775675.739988:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775675.739991:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775675.739993:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775675.739997:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.739999:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740002:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740004:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775675.740007:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775675.740010:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740012:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775675.740015:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775675.740017:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775675.740020:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740021:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775675.740023:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775675.740031:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775775675.740034:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775675.740037:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775675.740038:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740041:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775675.740043:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775675.740047:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775675.740054:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775675.740059:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775675.740066:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099655552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775675.740074:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775675.740078:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775675.740081:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099655552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775675.740103:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775675.740107:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775675.740111:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775675.740113:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775675.740114:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775675.740118:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775675.740121:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17391 00000400:00000200:1.0:1775775675.740126:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775675.740129:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775675.740132:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775675.740136:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740139:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740142:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740145:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775675.740148:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775675.740149:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775675.740153:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775675.740156:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775675.740160:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775675.740167:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775675.740176:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740181:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740215:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740220:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775675.740226:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775675.740230:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740233:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775675.740235:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775675.740241:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775675.740246:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740248:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775675.740253:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775675.740256:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740258:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775675.740264:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775775675.740268:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775675.740271:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775675.740273:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740277:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775675.740279:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775675.740282:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775675.740290:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775675.740296:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775675.740305:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099655680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775675.740321:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775675.740325:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775675.740335:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099655680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775675.740351:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775675.740355:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775675.740359:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775675.740361:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775675.740363:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775675.740367:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775675.740379:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17391 00000400:00000200:1.0:1775775675.740386:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775675.740389:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775675.740391:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775675.740393:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740395:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740399:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740404:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775675.740408:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775675.740411:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775675.740414:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775675.740415:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775675.740419:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775675.740424:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775675.740427:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740429:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740431:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740433:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775675.740435:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775675.740440:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740442:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775675.740445:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775675.740449:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775775675.740452:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775775675.740453:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740457:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.1:1775775675.740458:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000008:00100000:1.0:1775775675.740459:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775675.740463:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740467:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775675.740471:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775775675.740474:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775675.740479:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775675.740481:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740483:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775675.740486:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775675.740488:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775675.740492:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775775675.740498:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775675.740504:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099655808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775675.740515:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775675.740517:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775675.740521:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099655808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775675.740543:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775675.740545:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775675.740549:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775675.740551:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775675.740553:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775675.740555:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775675.740561:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17391 00000400:00000200:1.0:1775775675.740568:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775675.740574:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775675.740577:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775675.740579:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740584:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740585:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740586:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775675.740588:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775675.740589:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775675.740592:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775675.740595:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775675.740597:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775675.740601:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775675.740603:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740608:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740609:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740611:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775675.740613:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775675.740616:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740619:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775675.740621:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775675.740624:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775675.740629:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740630:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775675.740633:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775675.740635:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740640:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775675.740646:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775775675.740649:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775675.740652:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775675.740654:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740658:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775675.740660:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775675.740663:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775675.740668:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775675.740672:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775675.740680:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099655936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775675.740690:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775675.740693:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775675.740698:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099655936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775675.740710:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775675.740714:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775675.740721:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775675.740723:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775675.740725:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775675.740729:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775675.740734:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17391 00000400:00000200:1.0:1775775675.740739:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775675.740743:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775675.740746:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775675.740752:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740754:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740758:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740759:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775675.740760:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775675.740761:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775675.740763:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775675.740766:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775675.740770:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775675.740772:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775675.740815:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740820:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775675.740824:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740828:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775675.740833:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775675.740839:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775675.740844:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775675.740848:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775675.740853:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775675.740857:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740863:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775675.740866:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775675.740872:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740876:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775675.740882:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775775675.740886:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775675.740891:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775675.740894:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775675.740898:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775675.740901:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775675.740905:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775675.740912:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775675.740917:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775675.740927:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099656064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775675.740944:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775675.740949:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775675.740957:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099656064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775675.740972:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775675.740981:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17401) 00000800:00000200:2.1:1775775675.741076:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775675.741078:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775675.741285:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775675.741287:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775675.741497:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775675.741499:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775675.741674:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775675.741675:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000040:1.0:1775775677.019864:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312064757, at 4312064768 00000100:00000010:1.0:1775775677.019878:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775677.532006:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775677.532011:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775677.532015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775680.603988:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775680.603995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775680.604001:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775680.859874:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775680.859889:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775680.859892:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775680.859896:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775680.859901:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775680.859909:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17396 00000400:00000200:1.0:1775775680.859919:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775680.859926:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775680.859929:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775680.859933:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.859937:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.859939:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.859941:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775680.859948:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775680.859950:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775680.859952:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775680.859954:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775680.859956:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775680.859958:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775680.859960:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.859961:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.859962:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.859963:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775680.859966:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775680.859968:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775680.859969:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775680.859971:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775680.859973:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775680.859975:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.859977:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775680.859979:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775680.859986:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775680.859989:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775680.859992:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775680.859993:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.859995:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775680.859998:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775680.860001:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775680.860007:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775680.860012:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775680.860020:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099656192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775680.860032:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775680.860035:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775680.860039:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099656192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775680.860056:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775680.860059:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775680.860061:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775680.860062:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775680.860064:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775680.860066:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775680.860067:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17396 00000400:00000200:1.0:1775775680.860071:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775680.860073:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775680.860074:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775680.860076:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860078:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.860079:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860080:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775680.860082:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775680.860083:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775680.860085:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775680.860086:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775680.860087:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775680.860089:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775680.860091:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860105:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.860108:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860109:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775680.860112:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775680.860114:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860116:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775680.860118:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775680.860123:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775680.860126:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860128:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775680.860133:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775680.860137:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860139:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775680.860144:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775680.860147:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775680.860152:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775680.860154:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860159:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775680.860160:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775680.860164:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775680.860170:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775775680.860175:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775680.860184:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099656320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775680.860200:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775680.860205:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775680.860214:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099656320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775680.860232:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775680.860237:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775680.860244:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775680.860246:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775680.860248:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775680.860254:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775680.860258:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17396 00000400:00000200:1.0:1775775680.860265:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775680.860270:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775680.860274:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775680.860281:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860288:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.860293:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860299:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775680.860304:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775680.860307:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775680.860311:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775680.860315:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775680.860318:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775680.860320:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775680.860328:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860333:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.860338:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860342:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775680.860344:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775680.860347:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860355:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775680.860357:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775680.860361:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775680.860366:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860372:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775680.860375:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775680.860379:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860382:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775680.860388:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775680.860392:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775680.860397:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775680.860400:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860403:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775680.860404:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775680.860408:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775680.860416:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775775680.860420:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775680.860438:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099656448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775680.860451:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1775775680.860454:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775680.860455:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775680.860460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000040:1.0:1775775680.860462:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099656448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775680.860479:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775680.860481:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775680.860486:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775680.860490:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775680.860493:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775680.860496:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775680.860502:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17396 00000400:00000200:1.0:1775775680.860507:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775680.860509:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775680.860511:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775680.860516:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860520:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.860525:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860530:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775680.860535:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775680.860537:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775680.860540:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775680.860544:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775680.860548:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775680.860554:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775680.860570:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860571:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.860574:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860577:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775680.860579:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775680.860582:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860585:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775680.860587:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775680.860591:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775680.860596:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860598:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775680.860599:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775680.860603:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860605:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775680.860608:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775680.860612:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775680.860614:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775680.860615:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860620:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775680.860621:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775680.860622:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775680.860629:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775680.860633:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775680.860641:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099656576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775680.860662:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775680.860664:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775680.860668:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099656576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775680.860684:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775680.860687:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775680.860692:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775680.860693:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775680.860694:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775680.860695:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775680.860698:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17396 00000400:00000200:1.0:1775775680.860703:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775680.860706:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775680.860709:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775680.860714:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860715:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.860718:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860721:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775680.860724:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775680.860725:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775680.860728:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775680.860730:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775680.860734:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775680.860738:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775680.860743:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860746:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775680.860751:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860752:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775680.860757:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775680.860761:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775680.860766:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775680.860770:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775680.860773:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775680.860776:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860797:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775680.860799:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775680.860801:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860802:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775680.860804:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775775680.860806:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775680.860808:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775680.860809:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775680.860810:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775680.860811:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775680.860812:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775680.860815:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775680.860817:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775680.860821:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099656704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775680.860827:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775680.860831:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775680.860835:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099656704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775680.860850:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775680.860853:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17406) 00000800:00000200:2.1:1775775680.861054:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775680.861056:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775680.861318:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775680.861319:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775680.861507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775680.861508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775680.861675:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775680.861676:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000040:1.0:1775775682.139899:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312069877, at 4312069888 00000100:00000010:1.0:1775775682.139924:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775683.676030:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775683.676037:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775683.676057:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775685.980439:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775685.980445:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775685.981150:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775685.981168:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775685.981380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775685.981383:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775685.981621:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775685.981623:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775685.981927:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775685.981929:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775686.748072:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775686.748083:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775686.748088:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775687.259923:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312074997, at 4312075008 00000100:00000010:1.0:1775775687.259940:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775775689.819991:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775689.819996:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775689.819999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775691.100944:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775691.100953:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775691.101775:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775691.101778:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775691.102136:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775691.102139:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775691.102428:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775691.102431:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775691.102735:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775691.102738:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000040:1.0:1775775692.379849:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312080118, at 4312080128 00000100:00000010:1.0:1775775692.379862:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000020:00000001:1.0:1775775692.892070:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775692.892073:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775692.892075:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775695.963953:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775695.963956:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775695.963959:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775696.220421:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775696.220425:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775696.221114:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775696.221115:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775696.221349:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775696.221350:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775696.221520:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775696.221522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775696.221729:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775696.221731:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000040:1.0:1775775697.499962:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312085237, at 4312085248 00000100:00000010:1.0:1775775697.499985:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775699.035957:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775699.035961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775699.035964:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775701.340916:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775701.340923:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775701.341572:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775701.341574:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775701.341906:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775701.341908:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775701.342160:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775701.342162:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775701.342432:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775701.342434:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775775702.108063:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775702.108069:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775702.108074:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775702.619886:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312090358, at 4312090368 00000100:00000010:1.0:1775775702.619899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000020:00000001:1.0:1775775705.179952:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775705.179956:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775705.179959:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775706.460439:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775706.460444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775706.461124:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775706.461126:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775706.461428:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775706.461430:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775706.461747:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775706.461750:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775706.462054:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775706.462057:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000040:1.0:1775775707.739928:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312095477, at 4312095488 00000100:00000010:1.0:1775775707.739961:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000020:00000001:1.0:1775775708.252020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775708.252025:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775708.252030:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775711.324021:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775711.324025:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775711.324030:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775711.580831:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775711.581401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775711.581740:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775711.582109:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775711.582297:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775775712.859892:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312100598, at 4312100608 00000100:00000010:1.0:1775775712.859912:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503440c0. 00000020:00000001:1.0:1775775714.395980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775714.395984:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775714.395987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775716.700263:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775716.700267:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775716.700892:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775716.700894:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775716.701188:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775716.701191:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775716.701495:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775716.701498:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775716.701804:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775716.701807:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775717.468012:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775717.468015:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775717.468018:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775717.979865:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312105717, at 4312105728 00000100:00000010:1.0:1775775717.979884:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000020:00000001:1.0:1775775720.539996:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775720.540000:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775720.540003:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775721.820460:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775721.820465:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775721.821071:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775721.821073:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775721.821290:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775721.821292:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775721.821507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775721.821508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775721.821684:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775721.821686:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1775775723.099866:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312110837, at 4312110848 00000100:00000010:1.0:1775775723.099885:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000020:00000001:1.0:1775775723.611926:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775723.611929:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775723.611932:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775726.684008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775726.684012:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775726.684015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775726.940349:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775726.940354:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775775726.940994:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775726.940997:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775775726.941252:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775726.941255:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775775726.941554:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775726.941556:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775775726.941761:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775726.941763:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000040:1.0:1775775728.219880:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312115957, at 4312115968 00000100:00000010:1.0:1775775728.219895:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503442a0. 00000020:00000001:1.0:1775775729.756023:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775729.756027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775729.756031:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775732.060398:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775732.060405:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775732.061081:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775732.061083:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775732.061288:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775732.061290:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775732.061499:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775732.061500:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775732.061747:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775732.061749:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775775732.827944:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775732.827948:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775732.827952:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775733.339857:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312121077, at 4312121088 00000100:00000010:1.0:1775775733.339875:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000020:00000001:1.0:1775775735.900029:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775735.900033:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775735.900036:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775737.180487:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775737.180494:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775737.181141:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775737.181144:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775737.181353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775737.181355:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775737.181529:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775737.181541:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775775737.181746:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775737.181747:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000100:00000040:1.0:1775775738.459874:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312126197, at 4312126208 00000100:00000010:1.0:1775775738.459904:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775775738.971954:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775738.971958:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775738.971961:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775742.043965:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775742.043969:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775742.043972:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775742.299889:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775742.299900:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775742.299902:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775742.299906:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775742.299911:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775742.299918:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17457 00000400:00000200:1.0:1775775742.299927:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775742.299932:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775742.299935:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775742.299938:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.299940:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.299942:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.299943:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775742.299946:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775742.299947:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775742.299950:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775742.299952:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775742.299954:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775742.299955:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775742.299957:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.299959:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.299960:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.299961:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775742.299963:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775742.299966:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775742.299967:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775742.299970:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775742.299973:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775742.299975:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.299977:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775742.299979:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775742.299987:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775775742.299990:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775742.299994:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775742.299995:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.299997:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775742.299999:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775742.300003:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775742.300010:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775775742.300015:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775742.300047:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099663872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775742.300056:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775742.300060:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775742.300063:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099663872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775742.300081:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775742.300083:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775742.300085:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775742.300087:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775742.300088:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775742.300090:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775742.300098:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17457 00000400:00000200:1.0:1775775742.300106:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775742.300111:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775742.300114:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775742.300118:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300122:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.300128:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300133:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775742.300138:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775742.300141:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775742.300145:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775742.300149:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775742.300153:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775742.300159:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775742.300166:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300169:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.300174:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300179:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775742.300182:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775742.300186:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300189:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775742.300191:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775742.300197:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775742.300201:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300203:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775742.300207:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775742.300210:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300211:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775742.300217:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775775742.300219:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775742.300224:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775742.300225:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300227:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775742.300229:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775742.300232:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775742.300238:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775742.300243:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775742.300252:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099664000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775742.300266:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775742.300269:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775742.300274:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099664000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775742.300288:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775742.300290:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775742.300305:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775742.300308:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775742.300310:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775742.300315:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775742.300319:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17457 00000400:00000200:1.0:1775775742.300328:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775742.300333:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775742.300336:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775742.300342:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300345:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.300349:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300352:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775742.300355:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775742.300357:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775742.300358:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775742.300360:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775742.300362:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775742.300364:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775742.300371:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300374:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.300379:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300383:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775742.300386:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775742.300392:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300395:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775742.300399:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775742.300402:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775742.300407:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300412:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775742.300416:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.1:1775775742.300416:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775742.300420:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000008:00000001:1.0:1775775742.300422:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300426:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775742.300433:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775775742.300437:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775742.300443:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775742.300446:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300449:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775742.300452:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775742.300455:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775742.300463:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775742.300469:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775742.300480:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099664128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775742.300502:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775742.300504:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775742.300509:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099664128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775742.300522:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775742.300524:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775742.300529:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775742.300530:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775742.300531:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775742.300533:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775742.300538:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17457 00000400:00000200:1.0:1775775742.300543:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775742.300550:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775742.300552:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775742.300555:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300558:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.300560:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300562:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775742.300565:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775742.300567:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775742.300570:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775742.300573:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775742.300578:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775742.300585:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775742.300591:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300596:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.300603:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300607:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775742.300613:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775742.300621:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300628:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775742.300632:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775742.300637:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775742.300642:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300647:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775742.300650:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775742.300654:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300657:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775742.300661:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775775742.300666:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775742.300672:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775742.300676:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300681:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775742.300684:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775742.300687:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775742.300694:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775742.300701:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775742.300711:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099664256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775742.300729:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775742.300734:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775742.300742:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099664256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775742.300756:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775742.300758:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775742.300766:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775742.300769:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775742.300772:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775742.300775:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775742.300820:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17457 00000400:00000200:1.0:1775775742.300828:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775742.300835:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775742.300841:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775742.300846:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300850:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.300852:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300855:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775742.300858:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775742.300860:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775742.300862:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775742.300864:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775742.300866:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775742.300870:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775742.300876:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300878:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775742.300881:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300883:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775742.300886:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775742.300890:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775742.300893:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775742.300896:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775742.300899:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775742.300901:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300905:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775742.300906:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775742.300911:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300912:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775742.300915:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775775742.300916:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775742.300918:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775742.300921:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775742.300926:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775742.300928:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775742.300931:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775742.300936:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775742.300940:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775742.300946:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099664384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775742.300971:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775742.300976:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775742.300986:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099664384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775742.301008:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775742.301016:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17467) 00000800:00000200:2.1:1775775742.301097:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775742.301098:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775742.301286:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775742.301288:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775742.301498:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775742.301500:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775775742.301829:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775742.301832:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000100:00000040:1.0:1775775743.579853:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312131317, at 4312131328 00000100:00000010:1.0:1775775743.579867:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775745.115989:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775745.115993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775745.115998:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775747.420926:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775747.420946:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775775747.421678:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775747.421679:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775775747.421916:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775747.421918:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775775747.422115:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775747.422117:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775775747.422348:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775747.422351:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1775775748.188041:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775748.188045:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775748.188048:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775748.699933:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312136438, at 4312136448 00000100:00000010:1.0:1775775748.699967:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775751.260003:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775751.260006:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775751.260009:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775752.539918:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775752.539925:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775752.539927:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775752.539931:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775752.539934:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775752.539940:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17467 00000400:00000200:1.0:1775775752.539948:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775752.539953:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775752.539955:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775752.539958:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.539961:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.539962:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.539964:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775752.539967:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775752.539969:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775752.539971:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775752.539973:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775752.539975:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775752.539977:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775752.539979:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.539980:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.539982:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.539983:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775752.539985:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775752.539987:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775752.539989:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775752.539991:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775752.539993:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775752.539995:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.539997:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775752.539998:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775752.540006:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775775752.540008:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775752.540012:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775752.540013:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540015:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775752.540017:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775752.540020:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775752.540027:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775775752.540032:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775752.540039:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099665152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775752.540047:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775752.540050:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775752.540054:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099665152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775752.540071:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775752.540074:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775752.540076:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775752.540083:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775752.540085:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775752.540087:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775752.540089:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17467 00000400:00000200:1.0:1775775752.540092:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775752.540095:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775752.540097:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775752.540100:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540102:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.540104:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540105:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775752.540107:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775752.540108:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775752.540111:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775752.540113:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775752.540117:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775752.540121:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775752.540125:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540127:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.540131:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540134:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775752.540139:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775752.540144:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540146:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775752.540149:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775752.540154:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775752.540159:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540163:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775752.540169:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775752.540173:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540176:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775752.540180:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775775752.540183:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775752.540186:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775752.540187:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540190:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775752.540193:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775752.540195:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775752.540199:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775775752.540203:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775752.540209:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099665280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775752.540217:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775752.540222:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775752.540227:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099665280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775752.540246:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775752.540251:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775752.540260:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775752.540263:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775752.540266:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775752.540273:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775752.540278:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17467 00000400:00000200:1.0:1775775752.540287:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775752.540296:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775752.540301:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775752.540309:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540313:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.540319:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540324:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775752.540329:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775752.540331:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775752.540333:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775752.540334:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775752.540338:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775752.540342:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775752.540349:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540352:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.540356:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540359:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775752.540361:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775752.540364:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540366:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775752.540369:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775752.540374:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775752.540380:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540383:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.1:1775775752.540383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00100000:1.0:1775775752.540386:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775752.540388:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540389:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775752.540392:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775775752.540394:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775752.540398:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775752.540400:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775752.540400:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000100:00000001:1.0:1775775752.540404:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775752.540406:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775752.540410:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775752.540415:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775775752.540419:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775752.540427:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099665408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775752.540436:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775752.540439:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775752.540444:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099665408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775752.540472:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775752.540475:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775752.540480:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775752.540483:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775752.540485:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775752.540492:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775752.540497:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17467 00000400:00000200:1.0:1775775752.540508:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775752.540515:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775752.540518:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775752.540523:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540528:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.540530:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540533:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775752.540535:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775752.540537:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775752.540539:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775752.540542:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775752.540546:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775752.540550:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775752.540553:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540556:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.540559:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540563:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775752.540567:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775752.540571:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540574:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775752.540577:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775752.540580:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775752.540585:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540588:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775752.540592:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775752.540597:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540598:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775752.540611:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775775752.540615:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775752.540621:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775752.540623:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540625:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775752.540628:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775752.540631:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775752.540636:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775752.540641:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775752.540651:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099665536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775752.540675:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775752.540679:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775752.540686:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099665536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775752.540705:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775752.540709:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775752.540714:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775752.540717:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775752.540719:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775752.540722:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775752.540727:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17467 00000400:00000200:1.0:1775775752.540732:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775752.540738:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775752.540741:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775752.540747:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540750:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.540755:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540757:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775752.540761:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775752.540763:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775752.540766:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775752.540769:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775752.540773:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775752.540801:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775752.540804:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540806:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775752.540807:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540809:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775752.540811:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775752.540813:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775752.540816:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775752.540817:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775752.540820:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775752.540822:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540826:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775752.540829:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775752.540833:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540836:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775752.540842:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775775752.540859:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775752.540865:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775752.540867:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775752.540872:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775752.540875:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775752.540880:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775752.540889:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775752.540897:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775752.540905:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099665664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775752.540925:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775752.540928:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775752.540934:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099665664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775752.540951:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775752.540954:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17477) 00000800:00000200:2.1:1775775752.541031:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775752.541033:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775775752.541215:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775752.541217:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775775752.541392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775752.541393:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775775752.541631:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775752.541633:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000100:00000040:1.0:1775775753.819901:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312141557, at 4312141568 00000100:00000010:1.0:1775775753.819914:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775754.332014:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775754.332017:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775754.332020:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775757.403969:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775757.403972:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775757.403974:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775757.659919:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775757.659927:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775757.659929:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775757.659933:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775757.659937:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775757.659943:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17472 00000400:00000200:1.0:1775775757.659951:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775757.659956:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775757.659959:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775757.659961:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.659964:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.659965:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.659967:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775757.659970:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775757.659972:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775757.659975:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775757.659977:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775757.659978:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775757.659980:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775757.659982:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.659983:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.659984:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.659985:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775757.659987:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775757.659989:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775757.659991:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775757.659993:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775757.659995:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775757.659997:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.659999:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775757.660000:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775757.660008:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775775757.660010:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775757.660014:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775757.660015:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660017:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775757.660019:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775757.660023:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775757.660029:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775757.660034:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775757.660041:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099665792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775757.660050:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775757.660054:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775757.660057:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099665792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775757.660081:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775757.660083:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775757.660085:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775757.660087:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775757.660088:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775757.660090:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775757.660092:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17472 00000400:00000200:1.0:1775775757.660095:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775757.660098:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775757.660100:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775757.660103:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660106:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.660108:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660111:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775757.660113:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775757.660115:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775757.660117:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775757.660120:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775757.660122:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775757.660125:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775757.660129:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660131:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.660133:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660135:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775757.660137:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775757.660141:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660145:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775757.660149:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775757.660157:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775757.660160:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660165:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775757.660170:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775757.660175:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660179:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775757.660186:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775757.660191:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775757.660196:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775757.660199:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660204:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775757.660207:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775757.660210:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775757.660216:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775775757.660222:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775757.660230:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099665920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775757.660245:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775757.660250:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775757.660256:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099665920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775757.660267:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775757.660271:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775757.660278:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775757.660281:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775757.660284:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775757.660289:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775757.660294:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17472 00000400:00000200:1.0:1775775757.660304:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775757.660311:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775757.660316:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775757.660323:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660327:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.660330:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660334:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775757.660337:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775757.660338:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775757.660339:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775757.660341:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775757.660344:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775757.660347:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775757.660350:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660351:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.660352:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660354:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775757.660355:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775757.660358:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660362:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775757.660367:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775757.660371:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775757.660377:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660380:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775757.660385:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775757.660390:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660395:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775757.660402:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775757.660407:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775757.660413:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775757.660416:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660419:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775757.660422:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775757.660424:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775757.660427:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775757.660432:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775757.660440:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099666048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775757.660449:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775757.660451:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775757.660454:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099666048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775757.660460:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775757.660463:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775757.660468:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775757.660469:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775757.660470:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775757.660472:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775757.660476:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17472 00000400:00000200:1.0:1775775757.660481:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775757.660486:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775757.660491:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775757.660493:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660497:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.660500:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660502:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775757.660505:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775757.660507:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000800:00000200:2.1:1775775757.660507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775757.660508:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.1:1775775757.660513:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000040:1.0:1775775757.660519:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775757.660522:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775757.660526:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775757.660530:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660531:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.660534:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660536:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775757.660537:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775757.660539:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660540:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775757.660541:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775757.660543:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775757.660546:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660548:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775757.660550:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775757.660553:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660556:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775757.660560:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775757.660561:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775757.660563:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775757.660566:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660567:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775757.660568:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775757.660571:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775757.660574:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775757.660577:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775757.660583:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099666176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775757.660610:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775757.660615:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775757.660624:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099666176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775757.660655:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775757.660659:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775757.660665:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775757.660666:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775757.660669:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775757.660672:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775757.660677:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17472 00000400:00000200:1.0:1775775757.660687:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775757.660692:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775757.660697:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775757.660700:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660703:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.660708:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660710:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775757.660713:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775757.660714:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775757.660718:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775757.660721:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775757.660726:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775757.660729:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775757.660734:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660739:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775757.660744:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660747:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775757.660750:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775757.660772:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775757.660774:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775757.660776:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775757.660801:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775757.660803:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660804:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775757.660807:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775757.660808:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660810:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775757.660814:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775757.660815:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775757.660818:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775757.660820:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775757.660823:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775757.660825:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775757.660828:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775757.660833:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775775757.660837:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775757.660842:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099666304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775757.660852:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775757.660856:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775757.660861:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099666304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775757.660871:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775757.660876:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17483) 00000800:00000200:2.1:1775775757.661171:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775757.661174:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775757.661421:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775757.661424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775757.661638:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775757.661640:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775757.661876:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775757.661878:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000040:1.0:1775775758.939998:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312146677, at 4312146688 00000100:00000010:1.0:1775775758.940055:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775760.476000:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775760.476005:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775760.476008:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775762.780464:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775762.780468:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775762.781109:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775762.781111:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775762.781295:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775762.781308:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775762.781531:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775762.781533:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775762.781890:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775762.781893:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775763.547938:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775763.547941:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775763.547944:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775764.059871:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312151797, at 4312151808 00000100:00000010:1.0:1775775764.059884:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775766.620027:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775766.620031:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775766.620033:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775767.900862:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775767.900868:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775767.901879:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775767.901881:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775767.902065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775767.902066:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775767.902300:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775767.902302:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775767.902600:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775767.902613:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000040:1.0:1775775769.179948:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312156918, at 4312156928 00000100:00000010:1.0:1775775769.179957:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775775769.691940:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775769.691944:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775769.691948:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775772.763959:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775772.763962:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775772.763964:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775773.020501:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775773.020506:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775773.021343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775773.021346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775773.021710:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775773.021712:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775773.022023:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775773.022024:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775775773.022271:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775773.022284:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000040:1.0:1775775774.299886:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312162037, at 4312162048 00000100:00000010:1.0:1775775774.299900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775775775.835989:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775775.835994:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775775.835998:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775778.139906:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775778.139914:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775778.139916:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775778.139920:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775778.139923:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775778.139929:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17493 00000400:00000200:1.0:1775775778.139937:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775778.139941:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775778.139944:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775778.139946:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.139949:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.139950:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.139952:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775778.139955:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775778.139956:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775778.139958:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775778.139960:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775778.139962:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775778.139963:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775778.139965:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.139967:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.139968:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.139970:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775778.139972:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775778.139974:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775778.139976:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775778.139978:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775778.139980:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775778.139982:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.139984:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775778.139985:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775778.139993:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775775778.139995:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775778.139998:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775778.139999:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140001:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775778.140003:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775778.140007:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775778.140013:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775775778.140018:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775778.140025:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099668352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775778.140033:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775778.140037:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775778.140040:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099668352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775778.140057:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775778.140060:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775778.140062:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775778.140063:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775778.140064:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775778.140066:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775778.140068:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17493 00000400:00000200:1.0:1775775778.140070:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775778.140072:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775778.140074:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775778.140076:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140077:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.140079:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140081:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775778.140083:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775778.140084:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775778.140085:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775778.140086:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775778.140088:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775778.140090:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775778.140092:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140093:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.140095:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140098:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775778.140100:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775778.140103:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140106:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775778.140108:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775778.140113:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775778.140118:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140122:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775778.140127:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775778.140133:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140136:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775778.140139:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775775778.140142:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775778.140146:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775778.140149:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140152:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775778.140154:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775778.140155:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775778.140161:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775775778.140165:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775778.140172:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099668480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775778.140184:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775778.140186:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775778.140189:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099668480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775778.140198:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775778.140201:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775778.140205:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775778.140208:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775778.140209:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775778.140211:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775778.140216:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17493 00000400:00000200:1.0:1775775778.140223:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775778.140234:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775778.140239:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775778.140244:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140248:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.140250:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140252:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775778.140257:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775778.140258:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775778.140259:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775778.140261:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775778.140262:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775778.140267:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775778.140272:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140274:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.140278:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140281:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775778.140282:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775778.140288:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140291:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775778.140293:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775778.140297:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775778.140299:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140302:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775778.140305:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775778.140309:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140313:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775778.140318:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775775778.140323:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775778.140325:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775778.140326:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140330:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775778.140332:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775778.140335:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775778.140341:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775775778.140345:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775778.140352:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099668608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775778.140362:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1775775778.140367:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775778.140370:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775778.140375:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099668608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775778.140381:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775778.140383:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775778.140387:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775778.140390:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775778.140394:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775778.140411:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775778.140416:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17493 00000400:00000200:1.0:1775775778.140420:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775778.140425:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775778.140427:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775778.140430:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140434:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.140438:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140441:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775778.140444:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775778.140452:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775778.140455:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775778.140457:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775778.140459:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775778.140460:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775778.140466:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140470:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.140474:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140477:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775778.140482:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775778.140489:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140497:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775778.140500:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775778.140503:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775778.140507:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140511:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775778.140515:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775778.140522:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140527:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775778.140534:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775775778.140539:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775778.140544:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775778.140545:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140548:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775778.140550:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775778.140552:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775778.140559:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775775778.140564:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775778.140571:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099668736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775778.140580:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775778.140583:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775778.140591:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099668736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775778.140609:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775778.140615:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775778.140625:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775778.140628:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775778.140632:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775778.140637:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775778.140644:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17493 00000400:00000200:1.0:1775775778.140652:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775778.140657:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775778.140660:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775778.140665:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140669:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.140673:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140678:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775778.140681:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775778.140683:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775778.140686:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775778.140690:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775778.140695:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775778.140700:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775778.140708:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140714:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775778.140720:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140724:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775778.140731:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775778.140740:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775778.140745:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775778.140749:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775778.140754:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775778.140759:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775778.140765:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775775778.140771:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775778.140828:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775778.140834:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140838:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775778.140847:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775775778.140852:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775778.140857:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775778.140861:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775778.140866:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775778.140869:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775778.140874:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775778.140882:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775775778.140888:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775778.140897:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099668864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775778.140927:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775778.140932:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775778.140941:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099668864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775778.140976:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775778.140984:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17503) 00000800:00000200:2.1:1775775778.141044:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775778.141207:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775778.141376:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775778.907939:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775778.907942:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775778.907944:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775779.419881:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312167157, at 4312167168 00000100:00000010:1.0:1775775779.419904:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775781.979996:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775781.980000:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775781.980003:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775783.259875:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775783.259883:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775783.259886:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775783.259891:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775783.259896:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775783.259903:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17498 00000400:00000200:1.0:1775775783.259914:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775783.259922:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775783.259926:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775783.259930:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.259934:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.259937:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.259940:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775783.259944:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775783.259946:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775783.259949:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775783.259952:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775783.259955:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775783.259958:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775783.259961:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.259963:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.259966:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.259967:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775783.259970:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775783.259974:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775783.259976:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775783.259979:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775783.259983:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775783.259986:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.259988:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775783.259991:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775783.259998:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775775783.260003:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775783.260006:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775783.260008:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260011:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775783.260014:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775783.260018:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775783.260026:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775775783.260032:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775783.260042:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099668992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775783.260053:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775783.260058:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775783.260063:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099668992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775783.260092:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775783.260096:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775783.260101:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775783.260102:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775783.260104:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775783.260107:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775783.260109:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17498 00000400:00000200:1.0:1775775783.260114:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775783.260117:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775783.260120:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775783.260123:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260125:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.260127:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260129:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775783.260132:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775783.260134:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775783.260135:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775783.260137:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775783.260139:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775783.260141:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775783.260143:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260145:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.260147:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260148:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775783.260151:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775783.260153:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260156:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775783.260158:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775783.260163:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775783.260165:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260167:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775783.260172:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775783.260175:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260177:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775783.260181:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775775783.260184:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775783.260187:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775783.260188:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260190:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775783.260192:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775783.260193:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775783.260198:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775775783.260201:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775783.260207:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099669120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775783.260214:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775783.260217:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775783.260220:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099669120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775783.260228:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775783.260230:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775783.260233:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775783.260235:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775783.260236:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775783.260239:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775783.260241:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17498 00000400:00000200:1.0:1775775783.260245:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775783.260248:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775783.260251:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775783.260255:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260256:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.260258:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260261:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775783.260263:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775783.260264:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775783.260266:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775783.260268:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775783.260270:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775783.260272:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775783.260275:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260277:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.260278:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260280:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775783.260282:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775783.260285:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260287:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775783.260289:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775783.260292:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775783.260293:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260295:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775783.260297:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775783.260300:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260302:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775783.260305:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775775783.260338:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775783.260341:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775783.260343:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260344:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775783.260346:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775783.260348:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775783.260352:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775775783.260355:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775783.260360:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099669248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775783.260373:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775783.260375:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775783.260378:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099669248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775783.260386:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775783.260388:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775783.260392:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775783.260394:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775783.260395:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775783.260398:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775783.260400:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17498 00000400:00000200:1.0:1775775783.260405:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775783.260408:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775783.260410:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775783.260418:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260421:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.260422:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260424:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775783.260426:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775783.260427:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775783.260429:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775783.260431:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775783.260433:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775783.260434:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775783.260437:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260438:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.260440:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260442:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775783.260444:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775775783.260447:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260449:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775783.260450:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775783.260453:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775783.260455:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260457:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775783.260459:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775783.260461:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260463:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775783.260466:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775775783.260469:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775783.260471:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775783.260472:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260474:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775783.260475:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775783.260477:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775783.260480:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775783.260484:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775783.260489:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099669376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775783.260495:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775783.260497:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775783.260501:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099669376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775783.260509:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775783.260511:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775783.260515:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775783.260517:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775783.260518:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775783.260521:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775783.260524:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17498 00000400:00000200:1.0:1775775783.260528:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775783.260531:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775783.260533:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775783.260536:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260538:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.260540:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260542:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775783.260544:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775783.260545:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775783.260547:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775783.260549:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775783.260551:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775783.260553:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775783.260556:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260558:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775783.260560:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260562:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775783.260564:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775783.260567:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775783.260569:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775783.260571:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775783.260574:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775783.260576:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260578:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775783.260580:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775783.260583:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260584:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775783.260588:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775775783.260591:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775783.260593:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775783.260594:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775783.260596:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775783.260598:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775783.260599:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775783.260603:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775783.260606:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775783.260611:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099669504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775783.260618:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775783.260621:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775783.260625:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099669504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775783.260632:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775783.260637:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17508) 00000800:00000200:2.1:1775775783.261165:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775783.261168:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775783.261549:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775783.261552:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775783.261978:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775783.261980:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775783.262233:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775783.262235:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775783.262492:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775783.262494:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000040:1.0:1775775784.539857:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312172278, at 4312172288 00000100:00000010:1.0:1775775784.539866:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775775785.052000:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775785.052004:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775785.052008:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775788.123991:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775788.123995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775788.123999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775788.380413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775788.380417:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775788.381018:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775788.381019:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775788.381198:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775788.381199:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775788.381428:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775788.381430:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775788.381700:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775788.381713:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1775775789.659866:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312177397, at 4312177408 00000100:00000010:1.0:1775775789.659884:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775791.195988:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775791.195990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775791.195993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775792.155889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775792.155895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775792.155896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775792.155901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775792.155904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775792.155907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775792.155909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775792.155911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775792.155913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775793.179896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775793.179904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775793.179906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775793.179912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775793.179916:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775793.179920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775793.179923:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775793.179925:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775793.179928:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775793.500507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775793.500511:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775793.501211:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775793.501213:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775793.501506:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775793.501509:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775793.501822:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775793.501825:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775793.502237:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775793.502239:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1775775794.203900:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775794.203906:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775794.203908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775794.203914:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775794.203917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775794.203921:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775794.203924:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775794.203926:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775794.203929:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775794.267988:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775794.267992:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775794.267996:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775794.779885:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312182517, at 4312182528 00000100:00000010:1.0:1775775794.779901:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:1.0:1775775795.227885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775795.227890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775795.227892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775795.227898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775795.227902:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775795.227906:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775795.227908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775795.227911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775795.227914:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775796.251874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775796.251880:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775796.251882:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775796.251887:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775796.251890:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775796.251893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775796.251895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775796.251897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775796.251899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775797.275907:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775797.275919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775797.275921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775797.275927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775797.275930:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775797.275934:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775797.275936:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775797.275939:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775797.275941:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775797.340060:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775797.340086:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775797.340089:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775798.299867:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775798.299876:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775798.299878:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775798.299884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775798.299888:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775798.299891:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775798.299894:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775798.299896:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775798.299898:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775775798.620516:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775798.620522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775798.621019:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775798.621021:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775798.621233:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775798.621235:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775798.621413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775798.621415:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775775798.621701:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775798.621703:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000100:00000001:1.0:1775775799.323939:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775799.323948:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775799.323950:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775799.323957:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775799.323962:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775799.323967:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775799.323970:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775799.323973:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775799.323977:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775799.899959:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312187637, at 4312187648 00000100:00000010:1.0:1775775799.899999:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:1.0:1775775800.347870:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775800.347877:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775800.347880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775800.347885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775800.347890:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775800.347894:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775800.347897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775800.347899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775800.347902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775800.412001:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775800.412003:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775800.412006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775801.371898:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775801.371904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775801.371905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775801.371909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775801.371912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775801.371916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775801.371918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775801.371920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775801.371921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775802.395872:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775802.395880:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775802.395882:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775802.395887:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775802.395891:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775802.395895:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775802.395897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775802.395900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775802.395902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775803.419920:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775803.419930:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775803.419933:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775803.419940:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775803.419944:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775803.419948:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775803.419951:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775803.419953:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775803.419973:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775803.484054:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775803.484059:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775803.484063:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775803.740639:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775803.740645:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775803.741286:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775803.741308:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775803.741643:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775803.741646:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775803.741988:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775803.741991:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775775803.742290:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775803.742293:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000040:1.0:1775775805.019923:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312192758, at 4312192768 00000100:00000010:1.0:1775775805.019958:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775775806.556060:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775806.556063:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775806.556066:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775808.860759:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775808.860774:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775808.861432:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775808.861435:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775808.861758:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775808.861761:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775808.862122:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775808.862125:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775775808.862440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775808.862443:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775775809.628028:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775809.628032:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775809.628034:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775810.139905:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312197878, at 4312197888 00000100:00000010:1.0:1775775810.139925:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775812.700073:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775812.700076:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775812.700079:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775775813.979929:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775813.979938:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775775813.979940:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775813.979944:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775813.979947:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775813.979952:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17529 00000400:00000200:1.0:1775775813.979960:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775813.979965:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775813.979968:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775813.979970:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.979973:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.979974:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.979976:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775813.979979:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775813.979980:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775775813.979983:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775813.979985:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775775813.979987:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775775813.979988:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775775813.979990:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.979991:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.979992:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.979994:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775813.979995:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775775813.979998:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775813.979999:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775813.980001:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775813.980003:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775813.980005:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980007:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775775813.980008:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775813.980017:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775775813.980019:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775775813.980022:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775813.980023:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980025:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775813.980027:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775813.980031:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775813.980037:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775775813.980042:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775813.980049:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099672832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775813.980057:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775813.980061:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775813.980064:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099672832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775813.980082:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775813.980086:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775813.980089:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775775813.980091:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775813.980092:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775813.980096:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775813.980098:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17529 00000400:00000200:1.0:1775775813.980102:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775813.980106:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775813.980108:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775813.980111:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980114:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.980115:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980118:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775813.980120:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775813.980122:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775775813.980123:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775813.980125:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775775813.980127:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775775813.980130:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775775813.980132:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980134:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.980136:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980138:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775813.980140:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775775813.980143:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980146:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775813.980148:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775813.980153:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775813.980155:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980157:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775813.980161:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775813.980164:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980167:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775813.980171:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775775813.980173:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775775813.980176:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775813.980177:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980179:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775813.980180:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775813.980182:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775813.980185:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775775813.980188:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775813.980192:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099672960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775813.980199:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775813.980201:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775813.980204:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099672960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775813.980213:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775813.980216:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775813.980220:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775775813.980221:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775813.980223:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775813.980226:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775813.980228:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17529 00000400:00000200:1.0:1775775813.980231:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775813.980234:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775813.980236:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775813.980239:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980241:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.980244:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980246:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775813.980248:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775813.980249:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775775813.980251:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775813.980253:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775775813.980255:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775775813.980257:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775775813.980260:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980262:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.980263:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980265:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775813.980267:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775775813.980270:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980277:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775813.980278:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775813.980281:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775813.980283:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980285:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775813.980287:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775775813.980289:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980291:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775813.980294:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775775813.980297:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775775813.980299:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775813.980301:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980302:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775813.980303:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775813.980305:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775813.980310:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775775813.980313:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775813.980317:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099673088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775813.980324:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775813.980326:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775813.980329:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099673088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775813.980336:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775813.980339:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775813.980342:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775775813.980343:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775813.980344:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775813.980346:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775813.980348:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17529 00000400:00000200:1.0:1775775813.980352:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775813.980354:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775813.980356:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775813.980359:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980361:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.980362:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980364:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775813.980366:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775813.980367:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775775813.980368:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775813.980370:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775775813.980372:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775775813.980374:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775775813.980377:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980379:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.980381:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980383:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775813.980386:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000800:00000200:2.1:1775775813.980389:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775813.980393:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00080000:1.0:1775775813.980396:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980399:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775813.980401:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775813.980403:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775813.980405:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980407:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775813.980409:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775813.980411:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980413:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775813.980417:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775775813.980419:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775775813.980422:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775813.980423:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980424:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775813.980426:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775813.980427:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775813.980431:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775775813.980434:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775813.980438:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099673216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775813.980446:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775813.980448:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775813.980451:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099673216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775813.980459:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775775813.980462:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775775813.980465:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775775813.980467:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775775813.980468:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775775813.980471:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775775813.980473:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17529 00000400:00000200:1.0:1775775813.980477:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775813.980480:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775775813.980482:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775775813.980485:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980487:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.980489:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980491:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775775813.980494:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775775813.980495:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775775813.980497:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775775813.980498:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775775813.980500:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775775813.980503:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775775813.980506:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980508:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775775813.980510:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980512:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775775813.980514:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775775813.980516:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775775813.980519:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775775813.980521:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775775813.980523:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775775813.980525:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980527:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775775813.980529:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775775813.980532:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980534:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775775813.980537:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775775813.980539:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775775813.980541:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775775813.980543:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775813.980545:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775775813.980546:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775775813.980548:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775775813.980552:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775775813.980554:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775813.980558:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099673344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775775813.980566:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775775813.980568:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775813.980571:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099673344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775813.980580:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775813.980584:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17539) 00000800:00000200:2.1:1775775813.980753:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775813.980755:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775813.981020:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775813.981022:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775813.981211:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775813.981225:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775775813.981399:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775813.981401:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000040:1.0:1775775815.259877:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312202997, at 4312203008 00000100:00000010:1.0:1775775815.259897:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775775815.771963:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775815.771965:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775815.771968:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775818.844038:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775818.844041:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775818.844044:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775819.100734:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775819.100742:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775819.101765:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775819.101771:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775819.102217:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775819.102219:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775819.102507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775819.102508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775775819.102734:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775819.102736:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1775775820.379939:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312208118, at 4312208128 00000100:00000010:1.0:1775775820.379968:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775775821.916002:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775821.916007:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775821.916011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775824.220417:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775824.220422:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775824.221211:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775824.221214:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775824.221534:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775824.221536:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775824.221865:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775824.221868:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775775824.222200:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775824.222203:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775824.988077:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775824.988081:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775824.988085:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775825.499895:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312213237, at 4312213248 00000100:00000010:1.0:1775775825.499914:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775828.059988:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775828.059992:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775828.059996:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775829.340386:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775829.340407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775829.341004:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775829.341006:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775829.341213:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775829.341215:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775829.341406:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775829.341409:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775775829.341642:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775829.341644:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000040:1.0:1775775830.619851:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312218357, at 4312218368 00000100:00000010:1.0:1775775830.619865:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775775831.132040:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775831.132054:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775831.132058:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775834.204064:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775834.204069:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775834.204074:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775834.461957:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775834.461968:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775834.462531:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775834.462534:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775834.462870:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775834.462872:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775834.463175:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775834.463178:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775775834.463485:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775834.463487:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000040:1.0:1775775835.739871:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312223479, at 4312223488 00000100:00000010:1.0:1775775835.739886:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775775837.276013:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775837.276017:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775837.276020:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775839.580389:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775839.580395:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775839.581117:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775839.581122:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775839.581345:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775839.581359:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775839.581541:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775839.581543:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775775839.581795:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775839.581797:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000020:00000001:1.0:1775775840.347995:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775840.347998:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775840.348001:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775840.859891:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312228597, at 4312228608 00000100:00000010:1.0:1775775840.859901:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775843.419980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775843.419984:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775843.419988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775844.700451:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775844.700745:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775844.701147:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775844.701366:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775844.701523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:1.0:1775775845.979890:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312233717, at 4312233728 00000100:00000010:1.0:1775775845.979900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775846.491951:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775846.491956:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775846.491959:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775849.563957:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775849.563961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775849.563965:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775849.820576:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775849.820583:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775849.821203:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775849.821210:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775849.821660:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775849.821662:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775849.822008:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775849.822010:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775775849.822252:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775849.822253:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000040:1.0:1775775851.099905:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312238837, at 4312238848 00000100:00000010:1.0:1775775851.099915:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775852.635997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775852.636002:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775852.636006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775854.940507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775854.940513:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775854.941101:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775854.941103:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775854.941320:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775854.941333:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775854.941607:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775854.941610:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775775854.941926:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775854.941929:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775855.707967:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775855.707970:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775855.707974:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775856.219913:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312243957, at 4312243968 00000100:00000010:1.0:1775775856.219924:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775775858.779930:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775858.779933:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775858.779936:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775860.060481:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775860.060489:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775860.061113:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775860.061114:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775860.061338:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775860.061340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775860.061517:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775860.061519:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775775860.061723:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775860.061724:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000040:1.0:1775775861.339916:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312249077, at 4312249088 00000100:00000010:1.0:1775775861.339930:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775775861.852044:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775861.852049:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775861.852053:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775864.924023:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775864.924027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775864.924029:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775775865.180564:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775865.180571:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775865.181174:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775865.181176:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775865.181369:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775865.181371:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775865.181595:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775865.181597:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775775865.181778:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775775865.181814:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000040:1.0:1775775866.459953:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312254197, at 4312254208 00000100:00000010:1.0:1775775866.459973:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775775867.995952:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775867.995955:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775867.995959:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775870.300760:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775870.300767:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775775870.301661:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775870.301663:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775775870.301898:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775870.301900:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775775870.302114:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775870.302116:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775775870.302290:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775870.302291:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775775871.067999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775871.068003:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775871.068006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775871.579903:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312259318, at 4312259328 00000100:00000010:1.0:1775775871.579922:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000020:00000001:1.0:1775775872.091959:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775872.091962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775872.091965:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775874.139936:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775874.139940:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775874.139944:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775875.164040:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775875.164044:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775875.164046:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775875.421802:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775875.421806:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775775875.422208:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775875.422212:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775775875.422507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775875.422510:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775775875.422954:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775875.422956:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775775875.423251:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775875.423254:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000100:00000040:1.0:1775775876.699915:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312264439, at 4312264448 00000100:00000010:1.0:1775775876.699932:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000020:00000001:1.0:1775775877.211999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775877.212002:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775877.212005:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775878.235942:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775878.235945:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775878.235949:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775880.284020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775880.284027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775880.284032:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775880.540471:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775880.540478:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775775880.541060:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775880.541063:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775775880.541377:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775880.541379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775775880.541691:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775880.541694:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775775880.542013:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775880.542016:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1775775881.307983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775881.307989:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775881.307994:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775881.819957:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312269557, at 4312269568 00000100:00000010:1.0:1775775881.819986:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000020:00000001:1.0:1775775883.355990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775883.355996:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775883.356000:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775884.379980:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775884.379983:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775884.379986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775885.660389:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775885.660394:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775775885.660956:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775885.660957:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775775885.661139:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775885.661141:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775775885.661333:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775885.661334:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775775885.661505:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775885.661507:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775775886.427945:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775886.427948:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775886.427951:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775886.939870:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312274677, at 4312274688 00000100:00000010:1.0:1775775886.939882:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000020:00000001:1.0:1775775887.451991:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775887.451996:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775887.451999:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775889.499981:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775889.499983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775889.499986:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775890.523993:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775890.523996:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775890.523999:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775890.780603:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775890.780609:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775890.781490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775890.781492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775890.781821:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775890.781824:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775890.782065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775890.782067:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775890.782331:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775890.782333:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000040:1.0:1775775892.059915:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312279797, at 4312279808 00000100:00000010:1.0:1775775892.059932:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000100:00000001:1.0:1775775892.507972:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775892.507981:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775892.507983:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775892.507990:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775892.507994:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775892.507998:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775892.508001:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775892.508004:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775892.508006:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775892.571974:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775892.571977:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775892.571980:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775893.531877:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775893.531894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775893.531897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775893.531902:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775893.531906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775893.531910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775893.531913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775893.531915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775893.531919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775893.595954:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775893.595957:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775893.595960:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775894.557004:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775894.557011:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775894.557013:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775894.557019:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775894.557023:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775894.557028:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775894.557030:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775894.557033:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775894.557035:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775895.580063:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775895.580079:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775895.580082:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775895.580088:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775895.580092:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775895.580097:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775895.580100:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775895.580102:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775895.580105:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775895.643949:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775895.643952:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775895.643955:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775895.900460:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775895.900466:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775895.901130:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775895.901133:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775895.901450:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775895.901453:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775895.901749:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775895.901752:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775895.902170:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775895.902173:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1775775896.603869:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775896.603876:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775896.603878:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775896.603884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775896.603888:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775896.603892:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775896.603895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775896.603899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775896.603901:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775896.668038:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775896.668041:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775896.668043:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775897.179880:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312284917, at 4312284928 00000100:00000010:1.0:1775775897.179894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344360. 00000100:00000001:1.0:1775775897.627870:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775897.627877:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775897.627879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775897.627885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775897.627889:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775897.627893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775897.627896:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775897.627899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775897.627902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775898.651888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775898.651893:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775898.651895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775898.651906:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775898.651909:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775898.651912:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775898.651914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775898.651916:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775898.651918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775898.715969:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775898.715972:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775898.715974:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775899.675872:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775899.675877:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775899.675879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775899.675884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775899.675887:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775899.675890:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775899.675892:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775899.675895:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775899.675897:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775899.739949:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775899.739952:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775899.739955:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775900.699864:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775900.699871:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775900.699873:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775900.699878:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775900.699882:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775900.699886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775900.699889:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775900.699891:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775900.699903:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775775901.020383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775901.020388:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775901.021004:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775901.021005:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775901.021190:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775901.021192:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775901.021432:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775901.021434:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775775901.021646:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775901.021648:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775775901.723895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775901.723902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775901.723904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775901.723910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775901.723913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775901.723917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775901.723920:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775901.723923:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775901.723926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775901.787950:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775901.787953:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775901.787956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775902.299882:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312290037, at 4312290048 00000100:00000010:1.0:1775775902.299902:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000100:00000001:1.0:1775775902.747882:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775902.747887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775902.747889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775902.747893:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775902.747896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775902.747899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775902.747901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775902.747903:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775902.747906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775902.811989:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775902.811992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775902.811995:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775903.771906:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775903.771912:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775903.771913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775903.771917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775903.771920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775903.771923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775903.771925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775903.771927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775903.771930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775904.795845:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775904.795850:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775904.795851:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775904.795856:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775904.795859:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775904.795862:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775904.795865:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775904.795866:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775904.795868:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775904.860036:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775904.860040:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775904.860043:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775905.819894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775905.819900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775905.819901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775905.819905:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775905.819908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775905.819912:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775905.819914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775905.819916:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775905.819918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775905.883972:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775905.883976:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775905.883980:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775906.140393:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775906.140397:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775906.141031:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775906.141034:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775906.141390:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775906.141391:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775906.141645:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775906.141648:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775775906.141868:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775906.141870:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1775775906.843841:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775906.843846:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775906.843848:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775906.843853:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775906.843855:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775906.843858:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775906.843860:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775906.843863:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775906.843864:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775907.419869:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312295157, at 4312295168 00000100:00000010:1.0:1775775907.419882:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000100:00000001:1.0:1775775907.867905:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775907.867912:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775907.867914:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775907.867920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775907.867926:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775907.867931:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775907.867934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775907.867937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775907.867940:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775907.931983:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775907.931987:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775907.931991:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775908.955958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775908.955962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775908.955964:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775911.003982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775911.003985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775911.003988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775911.260390:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775911.260687:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775911.261057:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775911.261372:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775911.261943:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775912.027968:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775912.027971:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775912.027974:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775912.539914:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312300277, at 4312300288 00000100:00000010:1.0:1775775912.539929:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344d20. 00000020:00000001:1.0:1775775914.075993:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775914.075997:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775914.076000:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775775914.332055:0:301877:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775775914.332100:0:301877:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775775914.332110:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775914.332114:0:301877:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775775914.332119:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775775914.332123:0:301877:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775775914.332128:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775914.332132:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511b00. 00000400:00000200:1.0:1775775914.332137:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775914.332143:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775914.332148:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1bb0 00000400:00000010:1.0:1775775914.332151:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1bb0. 00000100:00000001:1.0:1775775914.332156:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775914.332167:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454de300 x1862033099684992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332199:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775914.332204:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099684992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332213:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775914.332215:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775775914.332224:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775914.332227:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511700. 00000400:00000200:1.0:1775775914.332231:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775914.332236:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775914.332239:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e15d8 00000400:00000010:1.0:1775775914.332241:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e15d8. 00000100:00000001:1.0:1775775914.332243:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775914.332249:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dfb80 x1862033099685120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332258:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775914.332269:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dfb80 x1862033099685120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332293:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775914.332299:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775775914.332309:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775914.332313:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511f00. 00000400:00000200:1.0:1775775914.332318:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775914.332324:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775914.332330:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e13b8 00000400:00000010:1.0:1775775914.332332:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e13b8. 00000100:00000001:1.0:1775775914.332339:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775914.332347:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dd180 x1862033099685248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332365:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775914.332372:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099685248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332389:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775914.332393:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775775914.332405:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775914.332410:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511400. 00000400:00000200:1.0:1775775914.332414:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775914.332421:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775914.332426:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e17f8 00000400:00000010:1.0:1775775914.332430:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e17f8. 00000100:00000001:1.0:1775775914.332435:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775914.332443:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dca80 x1862033099685376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332459:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775914.332465:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099685376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332479:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775914.332483:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775775914.332495:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775775914.332500:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511500. 00000400:00000200:1.0:1775775914.332505:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775775914.332513:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775775914.332518:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1a18 00000400:00000010:1.0:1775775914.332522:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1a18. 00000100:00000001:1.0:1775775914.332527:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775775914.332532:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454de680 x1862033099685504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332547:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775775914.332556:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099685504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775775936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775775914.332569:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775914.332572:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775775915.099958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775915.099962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775915.099967:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775916.381542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775916.381545:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775916.382238:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775916.382240:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775916.382423:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775916.382425:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775916.382655:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775916.382657:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775916.382862:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775916.382864:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775775917.147929:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775917.147931:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775917.147933:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775917.659907:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312305398, at 4312305408 00000100:00000010:1.0:1775775917.659922:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000020:00000001:1.0:1775775918.171942:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775918.171944:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775918.171947:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775920.220032:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775920.220035:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775920.220038:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775921.243973:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775921.243976:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775921.243979:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775921.500329:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775921.500334:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775921.500732:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775921.500733:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775921.500931:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775921.500932:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775921.501086:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775921.501088:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775921.501230:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775921.501231:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1775775922.779923:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312310517, at 4312310528 00000100:00000010:1.0:1775775922.779930:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000020:00000001:1.0:1775775923.291979:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775923.291983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775923.291987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775924.315973:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775924.315976:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775924.315979:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775926.363980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775926.363983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775926.363987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775926.620355:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775926.620359:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775926.620823:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775926.620825:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775926.621003:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775926.621022:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775926.621182:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775926.621184:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775775926.621357:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775926.621358:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000020:00000001:1.0:1775775927.387995:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775927.387999:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775927.388002:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775927.899862:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312315637, at 4312315648 00000100:00000010:1.0:1775775927.899880:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344d20. 00000020:00000001:1.0:1775775929.435935:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775929.435940:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775929.435950:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775930.459934:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775930.459936:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775930.459939:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775931.740328:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775931.740332:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775931.740952:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775931.740954:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775931.741145:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775931.741147:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775931.741331:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775931.741333:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775775931.741590:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775931.741592:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1775775932.507941:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775932.507944:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775932.507946:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775933.019904:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312320757, at 4312320768 00000100:00000010:1.0:1775775933.019915:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503447e0. 00000020:00000001:1.0:1775775933.531974:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775933.531977:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775933.531980:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775935.579932:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775935.579935:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775935.579937:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775936.603955:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775936.603959:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775936.603962:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775936.860348:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775936.860354:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775936.860916:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775936.860918:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775936.861112:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775936.861113:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775936.861344:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775936.861346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775775936.861573:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775936.861585:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000040:1.0:1775775938.139868:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312325877, at 4312325888 00000100:00000010:1.0:1775775938.139882:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000020:00000001:1.0:1775775938.651980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775938.651982:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775938.651984:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775939.676005:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775939.676009:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775939.676012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775941.723959:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775941.723963:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775941.723966:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775941.980375:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775941.980379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775775941.980903:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775941.980904:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775775941.981077:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775941.981079:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775775941.981284:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775941.981286:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775775941.981508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775941.981510:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775775942.747957:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775942.747960:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775942.747964:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775943.259915:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312330997, at 4312331008 00000100:00000010:1.0:1775775943.259930:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503442a0. 00000020:00000001:1.0:1775775944.796022:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775944.796025:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775944.796027:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775945.819967:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775945.819971:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775945.819974:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775947.100365:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775947.100369:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775775947.100952:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775947.100956:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775775947.101199:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775947.101201:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775775947.101394:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775947.101396:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775775947.101564:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775947.101565:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775775947.867956:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775947.867959:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775947.867963:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775948.379900:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312336117, at 4312336128 00000100:00000010:1.0:1775775948.379913:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000020:00000001:1.0:1775775948.891928:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775948.891931:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775948.891934:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775950.939990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775950.939993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775950.939996:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775951.963945:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775951.963948:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775951.963950:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775952.220416:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775952.220419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775775952.220984:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775952.220998:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775775952.221306:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775952.221308:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775775952.221528:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775952.221531:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775775952.221705:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775952.221706:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1775775953.499868:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312341237, at 4312341248 00000100:00000010:1.0:1775775953.499883:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503445a0. 00000020:00000001:1.0:1775775954.011940:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775954.011943:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775954.011947:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775955.035975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775955.035977:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775955.035980:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775957.084005:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775957.084008:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775957.084010:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775957.340353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775957.340358:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775957.340982:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775957.340984:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775957.341247:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775957.341249:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775957.341462:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775957.341464:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775775957.341611:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775957.341613:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775775958.108360:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775958.108363:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775958.108366:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775958.619922:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312346357, at 4312346368 00000100:00000010:1.0:1775775958.619938:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000020:00000001:1.0:1775775960.155977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775960.155981:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775960.155984:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775961.179968:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775961.179972:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775961.179975:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775962.460312:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775962.460315:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775962.460837:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775962.460838:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775962.461008:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775962.461010:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775962.461200:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775962.461211:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775775962.461449:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775962.461451:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775775963.227962:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775963.227964:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775963.227966:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775963.739912:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312351477, at 4312351488 00000100:00000010:1.0:1775775963.739929:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000020:00000001:1.0:1775775964.252013:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775964.252017:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775964.252020:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775966.299990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775966.299994:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775966.299997:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775967.324043:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775967.324046:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775967.324049:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775967.580637:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775967.580642:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775775967.581374:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775967.581377:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775775967.581587:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775967.581589:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775775967.581865:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775967.581867:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775775967.582323:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775967.582326:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000100:00000040:1.0:1775775968.859935:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312356597, at 4312356608 00000100:00000010:1.0:1775775968.859939:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000020:00000001:1.0:1775775969.372009:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775969.372012:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775969.372016:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775970.395960:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775970.395964:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775970.395967:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775972.443967:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775972.443969:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775972.443972:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775972.700315:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775972.700322:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775775972.700840:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775972.700842:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775775972.701040:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775972.701042:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775775972.701198:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775972.701199:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775775972.701365:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775972.701368:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000020:00000001:1.0:1775775973.467941:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775973.467944:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775973.467947:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775973.979909:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312361717, at 4312361728 00000100:00000010:1.0:1775775973.979923:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000020:00000001:1.0:1775775975.515998:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775975.516004:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775975.516008:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775976.540038:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775976.540041:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775976.540044:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775977.820315:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775977.820652:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775977.820946:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775977.821113:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775977.821250:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775775978.587990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775978.587993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775978.587994:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775979.099863:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312366837, at 4312366848 00000100:00000010:1.0:1775775979.099875:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c60. 00000020:00000001:1.0:1775775979.611998:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775979.612001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775979.612004:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775981.659956:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775981.659959:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775981.659962:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775982.683975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775982.683978:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775982.683981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775982.941282:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775982.941285:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775982.941629:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775982.941631:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775982.942254:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775982.942256:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775982.942618:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775982.942620:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775775982.942862:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775982.942864:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000040:1.0:1775775984.219904:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312371958, at 4312371968 00000100:00000010:1.0:1775775984.219922:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344360. 00000020:00000001:1.0:1775775984.731990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775984.731994:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775984.731997:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775985.755960:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775985.755963:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775985.755966:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775987.804031:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775987.804035:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775987.804037:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775988.060503:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775988.060508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775988.061253:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775988.061258:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775988.061534:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775988.061537:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775988.061754:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775988.061755:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775775988.062030:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775988.062032:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775775988.827960:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775988.827963:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775988.827965:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775989.339914:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312377077, at 4312377088 00000100:00000010:1.0:1775775989.339930:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000020:00000001:1.0:1775775990.875963:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775990.875966:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775990.875969:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775775991.900064:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775991.900067:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775991.900070:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775993.180380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775993.180383:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775775993.180901:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775993.180904:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775775993.181106:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775993.181108:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775775993.181321:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775993.181323:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775775993.181508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775993.181509:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775775993.947958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775993.947961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775993.947963:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775775994.459925:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312382197, at 4312382208 00000100:00000010:1.0:1775775994.459943:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000100:00000001:1.0:1775775994.907878:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775994.907883:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775994.907885:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775994.907888:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775994.907891:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775994.907894:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775994.907895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775994.907897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775994.907899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775994.971943:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775994.971946:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775994.971948:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775995.931901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775995.931908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775995.931910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775995.931915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775995.931918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775995.931922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775995.931925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775995.931927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775995.931930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775775996.955838:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775996.955842:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775996.955844:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775996.955847:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775996.955850:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775996.955852:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775996.955854:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775996.955856:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775996.955858:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775997.019904:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775997.019906:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775997.019909:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775775997.979862:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775997.979867:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775997.979868:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775997.979872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775997.979874:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775997.979877:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775997.979878:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775997.979880:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775997.979881:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775775998.044011:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775775998.044014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775775998.044016:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775775998.300436:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775998.300443:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775998.300921:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775998.300923:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775998.301114:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775998.301116:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775998.301296:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775998.301297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775775998.301473:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775775998.301475:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000001:1.0:1775775999.003874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775775999.003879:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775775999.003881:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775775999.003884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775775999.003887:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775775999.003890:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775999.003892:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775775999.003894:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775775999.003895:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775775999.579887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312387317, at 4312387328 00000100:00000010:1.0:1775775999.579900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503447e0. 00000100:00000001:1.0:1775776000.027933:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776000.027941:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776000.027943:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776000.027949:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776000.027952:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776000.027957:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776000.027959:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776000.027962:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776000.027964:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776000.091990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776000.091995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776000.091999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776001.051918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776001.051926:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776001.051928:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776001.051934:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776001.051937:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776001.051942:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776001.051945:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776001.051947:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776001.051949:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776001.115942:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776001.115946:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776001.115949:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776002.075893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776002.075900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776002.075901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776002.075907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776002.075911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776002.075915:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776002.075918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776002.075920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776002.075923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776003.099887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776003.099893:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776003.099894:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776003.099899:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776003.099901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776003.099905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776003.099913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776003.099915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776003.099917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776003.163954:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776003.163958:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776003.163962:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776003.420597:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776003.420604:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776003.421296:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776003.421299:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776003.421626:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776003.421628:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776003.421971:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776003.421975:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776003.422280:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776003.422283:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000100:00000001:1.0:1775776004.123873:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776004.123878:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776004.123880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776004.123884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776004.123887:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776004.123890:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776004.123892:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776004.123894:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776004.123896:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776004.187958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776004.187961:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776004.187964:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776004.699940:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312392438, at 4312392448 00000100:00000010:1.0:1775776004.699959:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000100:00000001:1.0:1775776005.147945:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776005.147954:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776005.147957:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776005.147964:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776005.147968:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776005.147972:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776005.147975:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776005.147977:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776005.147980:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776006.171831:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776006.171836:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776006.171838:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776006.171842:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776006.171845:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776006.171847:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776006.171849:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776006.171851:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776006.171853:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776006.235975:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776006.235979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776006.235983:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776007.195880:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776007.195886:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776007.195887:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776007.195892:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776007.195895:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776007.195898:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776007.195900:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776007.195902:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776007.195904:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776007.259982:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776007.259986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776007.259989:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776008.219913:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776008.219918:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776008.219920:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776008.219924:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776008.219927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776008.219930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776008.219933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776008.219934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776008.219937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776008.540426:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776008.540430:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776008.541117:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776008.541119:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776008.541424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776008.541426:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776008.541739:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776008.541742:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776008.542121:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776008.542122:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000100:00000001:1.0:1775776009.243982:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776009.243989:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776009.243991:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776009.243997:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776009.244001:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776009.244006:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776009.244008:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776009.244011:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776009.244013:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776009.307984:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776009.307989:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776009.307993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776009.819909:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312397557, at 4312397568 00000100:00000010:1.0:1775776009.819925:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000100:00000001:1.0:1775776010.267889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776010.267894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776010.267895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776010.267899:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776010.267902:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776010.267906:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776010.267907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776010.267909:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776010.267911:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776010.331990:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776010.331993:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776010.331995:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776011.291885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776011.291890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776011.291891:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776011.291895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776011.291898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776011.291901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776011.291903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776011.291904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776011.291906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776012.315868:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776012.315874:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776012.315875:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776012.315880:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776012.315882:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776012.315885:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776012.315887:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776012.315889:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776012.315891:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776012.379987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776012.379990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776012.379993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776013.339857:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776013.339862:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776013.339863:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776013.339868:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776013.339870:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776013.339874:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776013.339877:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776013.339879:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776013.339882:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776013.403964:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776013.403967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776013.403969:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776013.660364:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776013.660370:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776013.660978:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776013.660980:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776013.661239:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776013.661242:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776013.661510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776013.661512:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776013.661762:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776013.661764:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:1.0:1775776014.363862:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776014.363866:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776014.363868:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776014.363872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776014.363874:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776014.363877:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776014.363879:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776014.363881:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776014.363882:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776014.939912:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312402677, at 4312402688 00000100:00000010:1.0:1775776014.939929:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000100:00000001:1.0:1775776015.387879:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776015.387884:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776015.387885:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776015.387889:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776015.387892:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776015.387894:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776015.387896:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776015.387897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776015.387899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776015.451952:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776015.451955:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776015.451957:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776016.411868:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776016.411874:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776016.411876:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776016.411881:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776016.411884:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776016.411888:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776016.411890:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776016.411893:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776016.411895:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776016.475891:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776016.475894:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776016.475896:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776017.435864:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776017.435869:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776017.435870:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776017.435874:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776017.435876:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776017.435879:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776017.435880:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776017.435882:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776017.435884:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776018.459885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776018.459890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776018.459892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776018.459895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776018.459898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776018.459901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776018.459903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776018.459904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776018.459906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776018.780445:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776018.780448:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775776018.781007:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776018.781010:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775776018.781338:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776018.781340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775776018.781565:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776018.781567:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775776018.781815:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776018.781817:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1775776019.483851:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776019.483856:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776019.483857:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776019.483861:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776019.483864:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776019.483866:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776019.483868:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776019.483870:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776019.483871:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776019.547951:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776019.547955:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776019.547958:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776020.059928:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312407797, at 4312407808 00000100:00000010:1.0:1775776020.059950:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000100:00000001:1.0:1775776020.507885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776020.507890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776020.507892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776020.507896:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776020.507898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776020.507901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776020.507903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776020.507904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776020.507906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776021.531852:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776021.531857:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776021.531858:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776021.531862:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776021.531864:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776021.531867:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776021.531869:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776021.531871:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776021.531872:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776022.555891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776022.555898:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776022.555901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776022.555906:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776022.555910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776022.555913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776022.555916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776022.555919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776022.555922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776022.619930:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776022.619935:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776022.619939:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776023.579899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776023.579905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776023.579906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776023.579911:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776023.579915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776023.579918:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776023.579921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776023.579923:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776023.579926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776023.899924:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776023.899943:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776023.899949:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776023.899953:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776023.899957:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776023.899965:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17739 00000400:00000200:1.0:1775776023.899976:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776023.899983:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776023.899986:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776023.899989:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.899992:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.899994:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.899997:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776023.900000:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776023.900002:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776023.900005:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776023.900007:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776023.900009:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776023.900012:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776023.900015:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900017:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900018:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900020:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776023.900023:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776023.900026:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900029:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776023.900031:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776023.900035:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776023.900037:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900040:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776023.900042:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776023.900051:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776023.900054:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776023.900058:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776023.900059:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900062:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776023.900065:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776023.900068:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776023.900075:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776023.900081:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776023.900090:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099699072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776023.900099:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776023.900104:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776023.900109:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099699072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776023.900151:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776023.900155:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776023.900158:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776023.900159:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776023.900161:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776023.900163:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776023.900166:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17739 00000400:00000200:1.0:1775776023.900169:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776023.900173:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776023.900175:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776023.900178:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900181:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900183:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900185:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776023.900188:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776023.900189:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776023.900191:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776023.900194:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776023.900196:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776023.900200:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776023.900206:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900209:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900214:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900219:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776023.900224:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776023.900228:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900231:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776023.900234:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776023.900241:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776023.900246:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900250:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776023.900255:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776023.900259:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900262:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776023.900268:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776023.900274:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776023.900279:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776023.900281:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900285:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776023.900288:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776023.900290:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776023.900296:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776023.900301:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776023.900311:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099699200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776023.900340:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776023.900346:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776023.900353:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099699200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776023.900369:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776023.900373:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776023.900378:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776023.900382:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776023.900385:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776023.900390:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776023.900395:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17739 00000400:00000200:1.0:1775776023.900403:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776023.900410:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776023.900415:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776023.900422:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900426:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900429:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900432:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776023.900435:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776023.900437:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776023.900440:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776023.900444:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776023.900448:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776023.900452:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776023.900460:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900465:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900468:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900473:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776023.900478:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776023.900484:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900487:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776023.900491:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775776023.900492:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.1:1775776023.900495:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 02000000:00000001:1.0:1775776023.900497:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776023.900502:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900507:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776023.900510:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776023.900515:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900519:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776023.900525:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776023.900530:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776023.900534:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776023.900537:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900541:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776023.900544:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776023.900547:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776023.900553:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776023.900558:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776023.900567:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099699328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776023.900581:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776023.900584:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776023.900591:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099699328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776023.900604:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776023.900608:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776023.900614:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776023.900616:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776023.900617:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776023.900623:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776023.900628:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17739 00000400:00000200:1.0:1775776023.900634:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776023.900642:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776023.900646:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776023.900654:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900658:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900663:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900667:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776023.900672:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776023.900675:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776023.900679:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776023.900683:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776023.900687:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776023.900693:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776023.900701:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900705:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900709:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900713:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776023.900718:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776023.900724:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900730:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776023.900744:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776023.900748:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776023.900754:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900758:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776023.900760:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776023.900763:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900767:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776023.900773:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776023.900801:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776023.900806:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776023.900807:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.900809:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776023.900811:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776023.900813:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776023.900820:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776023.900825:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776023.900835:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099699456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776023.900855:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776023.900859:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776023.900866:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099699456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776023.900890:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776023.900896:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776023.900903:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776023.900906:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776023.900909:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776023.900916:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776023.900920:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17739 00000400:00000200:1.0:1775776023.900924:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776023.900928:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776023.900932:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776023.900937:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900941:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900946:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900951:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776023.900956:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776023.900958:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776023.900960:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776023.900962:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776023.900966:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776023.900970:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776023.900977:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900981:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776023.900987:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776023.900992:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776023.900996:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776023.901001:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776023.901007:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776023.901011:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776023.901015:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776023.901020:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.901023:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776023.901027:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776023.901032:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.901036:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776023.901041:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776023.901045:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776023.901049:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776023.901053:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776023.901058:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776023.901061:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776023.901064:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776023.901069:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776023.901075:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776023.901084:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099699584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776023.901099:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776023.901104:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776023.901111:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099699584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776023.901142:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776023.901149:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17749) 00000800:00000200:3.1:1775776023.901184:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776023.901186:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776023.901408:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776023.901410:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776023.901590:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776023.901593:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776023.901928:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776023.901931:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1775776024.604340:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776024.604349:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776024.604351:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776024.604357:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776024.604363:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776024.604368:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776024.604371:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776024.604375:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776024.604378:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776025.179925:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312412917, at 4312412928 00000100:00000010:1.0:1775776025.179943:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000100:00000001:1.0:1775776025.628183:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776025.628195:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776025.628197:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776025.628206:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776025.628210:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776025.628215:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776025.628218:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776025.628220:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776025.628224:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776025.691965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776025.691969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776025.691972:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776026.653028:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776026.653033:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776026.653035:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776026.653040:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776026.653043:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776026.653046:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776026.653048:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776026.653050:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776026.653052:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776028.700207:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776028.700213:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776028.700221:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776028.700227:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776028.700230:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776028.700234:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776028.700237:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776028.700239:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776028.700242:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776028.763965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776028.763968:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776028.763971:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776029.020316:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776029.020319:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776029.020888:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776029.020890:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776029.021130:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776029.021132:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776029.021320:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776029.021321:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776029.021474:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776029.021475:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1775776029.724054:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776029.724063:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776029.724065:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776029.724071:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776029.724076:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776029.724082:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776029.724085:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776029.724088:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776029.724092:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776030.299903:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312418037, at 4312418048 00000100:00000010:1.0:1775776030.299915:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000100:00000001:1.0:1775776030.747887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776030.747892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776030.747893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776030.747897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776030.747899:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776030.747903:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776030.747904:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776030.747906:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776030.747908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776031.771883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776031.771890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776031.771892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776031.771897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776031.771901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776031.771905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776031.771908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776031.771910:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776031.771913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776031.836003:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776031.836007:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776031.836010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776032.795916:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776032.795923:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776032.795925:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776032.795930:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776032.795933:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776032.795937:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776032.795939:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776032.795941:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776032.795943:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776033.819856:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776033.819866:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776033.819868:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776033.819875:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776033.819879:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776033.819884:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776033.819888:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776033.819890:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776033.819894:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776034.140246:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776034.140249:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776034.140591:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776034.140593:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776034.140830:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776034.140832:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776034.141004:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776034.141005:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776034.141157:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776034.141158:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775776034.843940:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776034.843949:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776034.843951:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776034.843958:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776034.843962:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776034.843967:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776034.843970:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776034.843973:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776034.843976:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776034.908044:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776034.908048:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776034.908052:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776035.419946:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312423157, at 4312423168 00000100:00000010:1.0:1775776035.419967:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000100:00000001:1.0:1775776035.867889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776035.867899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776035.867930:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776035.867938:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776035.867942:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776035.867947:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776035.867950:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776035.867954:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776035.867957:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776036.891921:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776036.891928:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776036.891930:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776036.891936:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776036.891939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776036.891943:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776036.891946:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776036.891948:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776036.891950:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776037.915910:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776037.915919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776037.915921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776037.915927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776037.915931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776037.915935:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776037.915938:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776037.915940:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776037.915942:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776037.980043:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776037.980047:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776037.980050:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776038.939925:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776038.939932:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776038.939934:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776038.939940:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776038.939943:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776038.939947:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776038.939950:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776038.939952:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776038.939954:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776039.259945:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776039.259960:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776039.259964:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776039.259968:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776039.259974:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776039.259984:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17754 00000400:00000200:1.0:1775776039.259996:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776039.260004:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776039.260008:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776039.260011:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260015:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.260017:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260020:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776039.260025:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776039.260027:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776039.260030:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776039.260034:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776039.260036:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776039.260038:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776039.260042:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260044:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.260046:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260047:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776039.260050:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776039.260053:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260056:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776039.260059:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776039.260063:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776039.260066:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260069:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776039.260072:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776039.260083:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776039.260088:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776039.260092:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776039.260094:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260097:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776039.260101:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776039.260106:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776039.260116:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776039.260125:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776039.260137:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099700992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776039.260148:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776039.260153:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776039.260158:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099700992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776039.260184:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776039.260188:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776039.260191:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776039.260192:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776039.260194:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776039.260197:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776039.260199:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17754 00000400:00000200:1.0:1775776039.260203:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776039.260206:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776039.260209:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776039.260213:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260215:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.260218:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260220:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776039.260223:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776039.260224:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776039.260226:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776039.260231:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776039.260234:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776039.260239:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776039.260244:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260248:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.260250:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260252:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776039.260254:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776039.260260:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260262:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776039.260264:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776039.260271:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776039.260275:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260279:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776039.260285:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776039.260294:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260299:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776039.260305:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776039.260310:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776039.260314:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776039.260317:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260321:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776039.260326:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776039.260329:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776039.260337:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775776039.260344:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776039.260357:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099701120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776039.260369:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776039.260373:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776039.260382:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099701120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776039.260423:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776039.260427:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776039.260433:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776039.260435:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776039.260437:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776039.260442:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776039.260448:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17754 00000400:00000200:1.0:1775776039.260456:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776039.260465:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776039.260468:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776039.260473:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260479:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.260482:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260484:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776039.260496:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776039.260499:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776039.260502:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776039.260505:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776039.260510:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776039.260515:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776039.260521:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260524:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.260526:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260532:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776039.260536:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776039.260546:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260553:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776039.260559:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776039.260569:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776039.260578:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260583:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776039.260586:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776039.260592:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260597:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776039.260602:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776039.260608:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776039.260610:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776039.260612:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260615:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776039.260620:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776039.260624:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776039.260632:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776039.260640:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776039.260652:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099701248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1775776039.260677:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776039.260686:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1775776039.260690:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776039.260696:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776039.260710:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099701248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776039.260724:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776039.260729:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776039.260737:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776039.260742:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776039.260746:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776039.260756:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776039.260766:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17754 00000400:00000200:1.0:1775776039.260775:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776039.260832:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776039.260836:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776039.260841:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260855:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.260860:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260863:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776039.260871:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776039.260874:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776039.260877:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776039.260880:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776039.260886:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776039.260888:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776039.260895:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260898:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.260904:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260909:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776039.260916:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776039.260922:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776039.260928:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776039.260933:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776039.260938:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776039.260941:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260945:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776039.260950:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776039.260954:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260955:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776039.260964:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776039.260969:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776039.260974:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776039.260976:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.260980:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776039.260983:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776039.260986:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776039.260992:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776039.260997:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776039.261007:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099701376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776039.261064:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776039.261068:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776039.261079:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099701376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776039.261112:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776039.261120:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776039.261131:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776039.261133:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776039.261136:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776039.261142:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776039.261157:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17754 00000400:00000200:1.0:1775776039.261173:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776039.261184:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776039.261192:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776039.261200:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.261205:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.261209:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.261213:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776039.261221:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776039.261223:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776039.261226:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776039.261230:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776039.261238:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776039.261244:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776039.261258:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.261268:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776039.261276:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776039.261285:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776039.261293:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776039.261308:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776039.261315:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776039.261320:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776039.261324:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776039.261328:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.261333:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776039.261337:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776039.261343:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.261349:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776039.261355:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776039.261360:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776039.261365:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776039.261369:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776039.261373:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776039.261376:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776039.261378:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776039.261384:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776039.261391:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776039.261401:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099701504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776039.261450:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776039.261454:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776039.261464:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099701504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776039.261490:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776039.261499:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17764) 00000800:00000200:3.1:1775776039.261600:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776039.261603:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776039.261911:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776039.261914:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776039.262168:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776039.262170:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776039.262466:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776039.262468:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1775776039.963930:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776039.963940:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776039.963941:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776039.963949:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776039.963954:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776039.963959:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776039.963962:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776039.963965:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776039.963968:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776040.539932:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312428278, at 4312428288 00000100:00000010:1.0:1775776040.539952:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503445a0. 00000100:00000001:1.0:1775776040.987897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776040.987904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776040.987906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776040.987911:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776040.987914:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776040.987919:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776040.987921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776040.987924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776040.987926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776041.052038:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776041.052043:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776041.052047:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776042.011932:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776042.011940:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776042.011942:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776042.011948:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776042.011952:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776042.011957:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776042.011959:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776042.011962:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776042.011965:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776043.035878:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776043.035884:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776043.035886:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776043.035891:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776043.035895:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776043.035899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776043.035901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776043.035904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776043.035906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776044.059901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776044.059909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776044.059911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776044.059916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776044.059920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776044.059924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776044.059927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776044.059929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776044.059931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776044.124001:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776044.124005:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776044.124009:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776044.380392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776044.380847:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776044.381021:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776044.381164:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776044.381291:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776045.083826:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776045.083832:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776045.083833:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776045.083837:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776045.083840:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776045.083842:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776045.083844:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776045.083847:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776045.083848:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776045.659925:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312433397, at 4312433408 00000100:00000010:1.0:1775776045.659941:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c60. 00000100:00000001:1.0:1775776046.107923:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776046.107933:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776046.107935:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776046.107941:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776046.107946:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776046.107951:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776046.107954:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776046.107956:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776046.107959:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776047.131905:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776047.131914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776047.131916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776047.131921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776047.131927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776047.131933:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776047.131936:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776047.131939:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776047.131942:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776047.195983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776047.195986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776047.195989:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775776047.452128:0:301876:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775776047.452189:0:301876:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775776047.452205:0:301876:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776047.452209:0:301876:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775776047.452214:0:301876:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776047.452219:0:301876:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775776047.452225:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776047.452230:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:1.0:1775776047.452236:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776047.452242:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776047.452250:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1660 00000400:00000010:1.0:1775776047.452253:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1660. 00000100:00000001:1.0:1775776047.452260:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776047.452272:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dc700 x1862033099701632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452313:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776047.452319:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099701632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452328:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776047.452330:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776047.452342:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776047.452344:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511100. 00000400:00000200:1.0:1775776047.452347:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776047.452350:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776047.452353:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1cc0 00000400:00000010:1.0:1775776047.452355:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1cc0. 00000100:00000001:1.0:1775776047.452358:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776047.452364:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dc380 x1862033099701760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452373:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776047.452376:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099701760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452382:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776047.452384:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776047.452388:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776047.452390:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511a00. 00000400:00000200:1.0:1775776047.452394:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776047.452397:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776047.452399:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1550 00000400:00000010:1.0:1775776047.452401:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1550. 00000100:00000001:1.0:1775776047.452404:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776047.452410:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dce00 x1862033099701888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452418:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776047.452420:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099701888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452427:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776047.452428:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776047.452433:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776047.452435:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511200. 00000400:00000200:1.0:1775776047.452438:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776047.452441:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776047.452444:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1880 00000400:00000010:1.0:1775776047.452446:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1880. 00000100:00000001:1.0:1775776047.452448:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776047.452453:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dc000 x1862033099702016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452472:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776047.452475:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099702016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452481:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776047.452482:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776047.452487:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776047.452489:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511d00. 00000400:00000200:1.0:1775776047.452491:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776047.452494:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776047.452497:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1c38 00000400:00000010:1.0:1775776047.452498:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1c38. 00000100:00000001:1.0:1775776047.452501:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776047.452507:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454ddc00 x1862033099702144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452516:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776047.452518:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099702144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776047.452524:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776047.452526:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776048.155897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776048.155904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776048.155906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776048.155911:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776048.155914:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776048.155917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776048.155919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776048.155921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776048.155923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.179870:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776049.179875:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776049.179876:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776049.179880:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776049.179883:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776049.179886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776049.179888:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776049.179889:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776049.179891:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776049.499960:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776049.499968:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776049.499970:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776049.499974:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776049.499977:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776049.499984:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17764 00000400:00000200:1.0:1775776049.499992:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776049.499997:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776049.500000:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776049.500002:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500004:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500006:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500008:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776049.500011:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776049.500012:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776049.500014:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776049.500015:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776049.500017:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776049.500018:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776049.500020:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500021:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500022:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500023:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776049.500025:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776049.500028:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500030:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776049.500033:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776049.500036:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776049.500039:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500041:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776049.500044:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776049.500050:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776049.500054:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776049.500057:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776049.500059:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500062:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776049.500064:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776049.500069:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776049.500076:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776049.500082:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776049.500102:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099702272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776049.500112:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776049.500116:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776049.500120:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099702272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776049.500149:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776049.500153:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776049.500156:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776049.500158:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776049.500159:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776049.500162:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776049.500164:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17764 00000400:00000200:1.0:1775776049.500168:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776049.500171:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776049.500174:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776049.500176:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500178:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500181:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500182:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776049.500185:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776049.500186:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776049.500188:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776049.500189:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776049.500191:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776049.500193:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776049.500196:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500197:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500199:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500200:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776049.500202:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776049.500205:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500207:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776049.500209:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776049.500214:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776049.500216:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500218:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776049.500222:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776049.500225:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500227:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776049.500231:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776049.500233:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776049.500235:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776049.500236:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500238:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776049.500240:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776049.500242:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776049.500245:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776049.500248:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776049.500252:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099702400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776049.500259:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776049.500261:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776049.500265:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099702400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776049.500272:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776049.500275:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776049.500278:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776049.500279:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776049.500280:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776049.500283:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776049.500290:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17764 00000400:00000200:1.0:1775776049.500296:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776049.500299:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776049.500301:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776049.500304:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500306:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500307:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500309:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776049.500310:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776049.500312:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776049.500313:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776049.500315:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776049.500317:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776049.500318:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776049.500321:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500322:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500324:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500326:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776049.500328:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776049.500330:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500332:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776049.500333:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776049.500336:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776049.500338:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500339:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776049.500341:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776049.500343:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500344:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776049.500348:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776049.500350:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776049.500351:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776049.500352:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500354:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776049.500355:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776049.500356:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776049.500359:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776049.500362:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776049.500366:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099702528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776049.500372:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776049.500374:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776049.500377:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099702528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776049.500383:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776049.500386:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776049.500388:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776049.500390:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776049.500391:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776049.500393:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776049.500397:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17764 00000400:00000200:1.0:1775776049.500400:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776049.500403:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776049.500405:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776049.500408:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500410:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500412:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500413:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776049.500416:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776049.500417:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776049.500418:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776049.500420:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776049.500422:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776049.500424:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776049.500427:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500430:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500432:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500436:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776049.500438:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776049.500443:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500446:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776049.500449:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776049.500451:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776049.500457:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500461:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776049.500465:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776049.500481:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500485:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776049.500489:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776049.500493:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776049.500497:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776049.500500:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500503:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776049.500505:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776049.500507:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776049.500512:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776049.500517:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776049.500525:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099702656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776049.500533:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776049.500536:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776049.500539:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099702656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776049.500556:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776049.500560:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776049.500568:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776049.500571:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776049.500572:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776049.500575:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776049.500578:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17764 00000400:00000200:1.0:1775776049.500581:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776049.500584:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776049.500586:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776049.500588:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500590:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500591:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500593:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776049.500594:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776049.500595:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776049.500597:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776049.500598:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776049.500600:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776049.500602:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776049.500604:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500606:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776049.500607:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500609:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776049.500611:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776049.500614:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776049.500616:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776049.500618:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776049.500620:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776049.500622:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500624:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776049.500626:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776049.500628:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500629:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776049.500639:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776049.500642:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776049.500643:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776049.500644:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776049.500646:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776049.500648:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776049.500649:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776049.500652:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776049.500655:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776049.500663:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099702784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776049.500688:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776049.500691:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776049.500699:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099702784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776049.500711:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776049.500718:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17774) 00000800:00000200:3.1:1775776049.500834:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776049.500837:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776049.501146:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776049.501148:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776049.501409:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776049.501411:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776049.501819:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776049.501821:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776049.502080:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776049.502083:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000001:1.0:1775776050.203934:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776050.203941:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776050.203944:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776050.203951:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776050.203955:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776050.203960:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776050.203963:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776050.203966:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776050.203968:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776050.267984:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776050.267987:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776050.267990:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776050.779903:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312438518, at 4312438528 00000100:00000010:1.0:1775776050.779920:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344180. 00000100:00000001:1.0:1775776051.227926:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776051.227937:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776051.227939:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776051.227946:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776051.227951:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776051.227956:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776051.227959:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776051.227962:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776051.227965:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776052.251919:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776052.251927:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776052.251929:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776052.251935:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776052.251938:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776052.251942:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776052.251944:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776052.251946:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776052.251948:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776053.275890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776053.275895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776053.275897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776053.275901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776053.275904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776053.275907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776053.275909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776053.275911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776053.275913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776053.340015:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776053.340020:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776053.340023:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776054.299876:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776054.299880:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776054.299881:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776054.299885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776054.299888:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776054.299891:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776054.299893:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776054.299895:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776054.299896:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776054.619919:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776054.619929:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776054.619932:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776054.619936:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776054.619940:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776054.619949:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17769 00000400:00000200:1.0:1775776054.619960:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776054.619967:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776054.619969:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776054.619978:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.619980:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.619982:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.619984:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776054.619987:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776054.619988:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776054.619991:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776054.619993:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776054.619995:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776054.619998:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776054.620002:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620003:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620005:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620006:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776054.620008:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776054.620010:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620011:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776054.620013:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776054.620016:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776054.620017:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620019:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776054.620021:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776054.620028:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776054.620031:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776054.620035:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776054.620036:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620038:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776054.620040:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776054.620043:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776054.620051:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776054.620056:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776054.620062:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099702912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776054.620069:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776054.620073:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776054.620076:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099702912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776054.620092:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776054.620094:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776054.620097:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776054.620098:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776054.620099:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776054.620101:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776054.620103:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17769 00000400:00000200:1.0:1775776054.620105:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776054.620106:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776054.620108:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776054.620110:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620111:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620112:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620113:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776054.620114:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776054.620115:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776054.620116:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776054.620117:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776054.620119:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776054.620120:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776054.620122:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620123:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620124:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620125:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776054.620127:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776054.620129:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620131:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776054.620132:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776054.620136:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776054.620137:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620138:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776054.620141:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776054.620144:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620145:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776054.620149:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776054.620151:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776054.620154:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776054.620155:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620156:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776054.620158:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776054.620160:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776054.620162:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776054.620167:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776054.620174:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099703040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776054.620183:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776054.620185:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776054.620189:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099703040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776054.620201:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776054.620204:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776054.620211:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776054.620214:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776054.620216:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776054.620221:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776054.620226:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17769 00000400:00000200:1.0:1775776054.620231:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776054.620236:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776054.620240:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776054.620245:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620250:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620252:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620256:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776054.620260:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776054.620261:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776054.620262:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776054.620266:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776054.620269:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776054.620271:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776054.620273:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620275:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620277:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620281:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776054.620284:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776054.620289:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620292:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776054.620295:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776054.620300:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776054.620304:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620307:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776054.620309:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776054.620312:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620313:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776054.620317:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776054.620320:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776054.620322:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776054.620323:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620324:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776054.620327:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776054.620328:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776054.620333:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776054.620338:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776054.620345:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099703168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776054.620367:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776054.620370:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776054.620377:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099703168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776054.620389:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776054.620392:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776054.620397:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776054.620399:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776054.620400:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776054.620402:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776054.620404:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17769 00000400:00000200:1.0:1775776054.620406:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776054.620410:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776054.620411:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776054.620412:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620417:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620421:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620425:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776054.620427:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776054.620429:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776054.620430:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776054.620432:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776054.620436:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776054.620441:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776054.620448:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620451:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620453:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620456:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776054.620460:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776054.620464:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620470:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776054.620471:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776054.620475:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776054.620479:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620481:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776054.620484:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776054.620486:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620488:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776054.620491:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000800:00000200:3.1:1775776054.620491:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:1.0:1775776054.620494:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776054.620498:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776054.620498:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776054.620498:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000100:00000001:1.0:1775776054.620502:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776054.620504:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776054.620505:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776054.620510:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776054.620513:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776054.620519:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099703296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776054.620527:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776054.620530:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776054.620535:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099703296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776054.620540:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776054.620543:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776054.620546:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776054.620548:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776054.620550:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776054.620552:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776054.620556:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17769 00000400:00000200:1.0:1775776054.620560:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776054.620565:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776054.620568:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776054.620571:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620572:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620573:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620574:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776054.620575:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776054.620589:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776054.620591:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776054.620593:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776054.620596:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776054.620600:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776054.620606:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620609:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776054.620612:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620613:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776054.620615:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776054.620617:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776054.620619:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776054.620621:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776054.620623:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776054.620626:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620628:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776054.620631:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776054.620634:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620637:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776054.620641:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776054.620644:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776054.620647:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776054.620649:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776054.620650:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776054.620651:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776054.620653:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776054.620655:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776054.620658:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776054.620664:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099703424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776054.620674:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776054.620677:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776054.620682:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099703424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776054.620691:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776054.620698:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17780) 00000800:00000200:3.1:1775776054.621023:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776054.621025:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775776054.621298:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776054.621300:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775776054.621528:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776054.621530:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775776054.621743:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776054.621745:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000100:00000001:1.0:1775776055.323858:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776055.323864:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776055.323866:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776055.323872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776055.323875:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776055.323879:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776055.323882:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776055.323884:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776055.323886:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776055.899907:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312443637, at 4312443648 00000100:00000010:1.0:1775776055.899922:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000100:00000001:1.0:1775776056.347897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776056.347903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776056.347905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776056.347910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776056.347913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776056.347917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776056.347919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776056.347921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776056.347924:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776056.412018:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776056.412022:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776056.412025:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776057.371924:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776057.371933:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776057.371935:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776057.371941:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776057.371946:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776057.371951:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776057.371954:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776057.371956:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776057.371960:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776058.395875:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776058.395881:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776058.395883:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776058.395888:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776058.395891:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776058.395895:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776058.395897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776058.395900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776058.395901:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776059.419880:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776059.419885:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776059.419886:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776059.419890:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776059.419892:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776059.419895:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776059.419897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776059.419898:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776059.419900:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776059.483997:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776059.484001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776059.484004:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776059.740571:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776059.740578:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776059.741417:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776059.741419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776059.741632:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776059.741633:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776059.741831:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776059.741834:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776059.742075:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776059.742077:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000100:00000001:1.0:1775776060.443875:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776060.443879:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776060.443880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776060.443884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776060.443886:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776060.443889:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776060.443892:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776060.443894:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776060.443896:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776061.019881:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312448757, at 4312448768 00000100:00000010:1.0:1775776061.019892:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000001:1.0:1775776061.467874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776061.467879:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776061.467881:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776061.467885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776061.467887:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776061.467890:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776061.467892:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776061.467893:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776061.467895:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776062.491855:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776062.491862:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776062.491864:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776062.491869:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776062.491881:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776062.491884:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776062.491890:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776062.491892:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776062.491895:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776062.555937:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776062.555939:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776062.555941:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776063.515884:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776063.515888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776063.515889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776063.515893:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776063.515896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776063.515899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776063.515900:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776063.515902:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776063.515904:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776064.539857:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776064.539864:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776064.539866:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776064.539871:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776064.539875:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776064.539878:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776064.539881:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776064.539883:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776064.539886:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776064.860381:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776064.860386:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776064.860984:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776064.860985:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776064.861182:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776064.861184:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776064.861392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776064.861393:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776064.861601:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776064.861603:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000100:00000001:1.0:1775776065.563858:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776065.563862:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776065.563863:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776065.563867:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776065.563869:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776065.563872:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776065.563874:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776065.563876:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776065.563878:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776065.627953:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776065.627955:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776065.627958:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776066.139849:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312453877, at 4312453888 00000100:00000010:1.0:1775776066.139861:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000100:00000001:1.0:1775776066.587846:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776066.587852:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776066.587853:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776066.587858:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776066.587862:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776066.587866:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776066.587868:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776066.587870:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776066.587872:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776067.611881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776067.611888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776067.611889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776067.611894:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776067.611896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776067.611900:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776067.611901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776067.611904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776067.611905:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776068.635883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776068.635888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776068.635889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776068.635894:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776068.635896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776068.635899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776068.635901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776068.635903:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776068.635905:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776068.699929:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776068.699932:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776068.699935:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776069.659891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776069.659897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776069.659899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776069.659903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776069.659907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776069.659910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776069.659913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776069.659915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776069.659918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776069.979898:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776069.979906:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776069.979908:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776069.979911:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776069.979915:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776069.979920:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17785 00000400:00000200:1.0:1775776069.979934:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776069.979939:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776069.979941:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776069.979943:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.979946:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.979947:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.979949:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776069.979952:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776069.979954:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776069.979956:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776069.979958:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776069.979959:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776069.979961:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776069.979962:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.979963:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.979964:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.979965:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776069.979967:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776069.979969:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776069.979970:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776069.979972:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776069.979974:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776069.979976:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.979977:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776069.979979:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776069.979985:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776069.979988:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776069.979990:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776069.979991:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.979993:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776069.979995:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776069.979998:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776069.980005:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776069.980010:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776069.980016:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099704832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776069.980022:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776069.980026:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776069.980029:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099704832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776069.980044:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776069.980047:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776069.980050:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776069.980052:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776069.980053:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776069.980055:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776069.980058:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17785 00000400:00000200:1.0:1775776069.980061:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776069.980064:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776069.980066:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776069.980068:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980070:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.980071:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980073:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776069.980075:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776069.980077:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776069.980078:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776069.980080:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776069.980081:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776069.980083:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776069.980086:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980088:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.980090:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980091:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776069.980094:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776069.980096:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980099:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776069.980101:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776069.980105:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776069.980107:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980109:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776069.980112:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776069.980115:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980116:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776069.980120:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776069.980122:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776069.980124:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776069.980125:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980127:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776069.980128:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776069.980130:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776069.980132:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776069.980135:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776069.980139:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099704960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776069.980145:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776069.980147:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776069.980150:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099704960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776069.980157:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776069.980160:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776069.980164:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776069.980165:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776069.980166:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776069.980169:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776069.980171:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17785 00000400:00000200:1.0:1775776069.980174:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776069.980176:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776069.980178:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776069.980181:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980182:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.980184:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980186:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776069.980188:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776069.980189:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776069.980190:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776069.980191:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776069.980193:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776069.980195:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776069.980197:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980199:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.980201:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980202:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776069.980205:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776069.980207:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980208:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776069.980210:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776069.980212:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776069.980214:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980216:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776069.980217:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776069.980219:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980221:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776069.980224:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776069.980226:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776069.980228:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776069.980229:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980231:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776069.980232:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776069.980233:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776069.980236:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776069.980239:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776069.980242:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099705088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776069.980249:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776069.980250:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776069.980254:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099705088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1775776069.980256:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776069.980259:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000001:1.0:1775776069.980261:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776069.980263:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776069.980266:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776069.980268:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776069.980269:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776069.980271:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776069.980273:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17785 00000400:00000200:1.0:1775776069.980276:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776069.980279:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776069.980281:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776069.980283:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980285:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.980287:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980289:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776069.980291:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776069.980292:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776069.980293:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776069.980295:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776069.980297:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776069.980299:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776069.980301:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980303:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.980305:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980307:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776069.980320:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776069.980323:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980325:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776069.980327:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776069.980329:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776069.980330:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980332:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776069.980334:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776069.980336:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980338:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776069.980341:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776069.980342:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776069.980344:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776069.980345:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980347:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776069.980348:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776069.980349:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776069.980352:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776069.980355:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776069.980359:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099705216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776069.980366:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776069.980367:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776069.980371:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099705216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776069.980378:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776069.980380:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776069.980383:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776069.980385:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776069.980387:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776069.980389:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776069.980391:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17785 00000400:00000200:1.0:1775776069.980394:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776069.980397:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776069.980399:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776069.980402:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980404:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.980406:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980407:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776069.980409:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776069.980410:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776069.980412:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776069.980413:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776069.980415:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776069.980417:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776069.980420:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980422:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776069.980424:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980426:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776069.980428:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776069.980430:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776069.980432:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776069.980433:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776069.980435:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776069.980437:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980439:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776069.980440:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776069.980442:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980443:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776069.980446:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776069.980448:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776069.980450:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776069.980452:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776069.980453:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776069.980454:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776069.980456:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776069.980459:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776069.980461:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776069.980464:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099705344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776069.980471:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776069.980472:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776069.980476:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099705344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776069.980482:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776069.980486:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17795) 00000800:00000200:3.1:1775776069.980593:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776069.980595:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775776069.980776:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776069.980793:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775776069.980968:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776069.980970:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775776069.981156:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776069.981159:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000001:1.0:1775776070.683881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776070.683887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776070.683888:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776070.683892:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776070.683894:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776070.683897:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776070.683899:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776070.683901:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776070.683903:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776071.259881:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312458997, at 4312459008 00000100:00000010:1.0:1775776071.259896:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000100:00000001:1.0:1775776071.707894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776071.707898:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776071.707900:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776071.707904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776071.707906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776071.707909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776071.707912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776071.707914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776071.707916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776071.771945:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776071.771949:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776071.771952:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776072.731878:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776072.731885:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776072.731886:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776072.731891:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776072.731894:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776072.731898:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776072.731901:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776072.731903:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776072.731906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776073.755925:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776073.755932:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776073.755933:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776073.755938:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776073.755942:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776073.755946:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776073.755948:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776073.755950:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776073.755953:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776074.779881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776074.779891:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776074.779893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776074.779900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776074.779905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776074.779909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776074.779913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776074.779916:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776074.779919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776074.843996:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776074.844000:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776074.844003:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776075.100501:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776075.100508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775776075.101157:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776075.101160:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775776075.101458:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776075.101460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775776075.101735:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776075.101737:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775776075.102029:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776075.102031:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:1.0:1775776075.803918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776075.803927:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776075.803931:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776075.803938:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776075.803943:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776075.803947:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776075.803950:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776075.803952:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776075.803955:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776076.379883:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312464117, at 4312464128 00000100:00000010:1.0:1775776076.379900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344360. 00000100:00000001:1.0:1775776076.827854:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776076.827861:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776076.827863:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776076.827869:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776076.827872:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776076.827876:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776076.827878:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776076.827881:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776076.827883:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776077.851883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776077.851887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776077.851889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776077.851892:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776077.851895:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776077.851898:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776077.851899:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776077.851909:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776077.851912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776077.916031:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776077.916033:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776077.916036:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776078.875942:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776078.875951:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776078.875953:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776078.875960:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776078.875965:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776078.875970:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776078.875973:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776078.875977:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776078.875980:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776079.899904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776079.899909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776079.899911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776079.899914:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776079.899917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776079.899920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776079.899922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776079.899924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776079.899925:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776080.220477:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776080.220482:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776080.220981:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776080.220983:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776080.221181:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776080.221183:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776080.221339:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776080.221341:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776080.221493:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776080.221494:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:1.0:1775776080.923919:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776080.923926:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776080.923928:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776080.923933:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776080.923936:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776080.923940:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776080.923942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776080.923944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776080.923946:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776080.988000:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776080.988004:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776080.988006:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776081.499895:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312469237, at 4312469248 00000100:00000010:1.0:1775776081.499913:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:1.0:1775776081.947893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776081.947900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776081.947902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776081.947908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776081.947911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776081.947914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776081.947916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776081.947919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776081.947921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776082.971916:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776082.971922:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776082.971924:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776082.971930:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776082.971933:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776082.971937:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776082.971940:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776082.971942:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776082.971944:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776083.995824:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776083.995831:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776083.995833:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776083.995839:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776083.995842:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776083.995845:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776083.995847:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776083.995850:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776083.995851:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776084.059983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776084.059987:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776084.059991:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776085.019906:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776085.019915:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776085.019917:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776085.019924:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776085.019928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776085.019934:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776085.019936:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776085.019939:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776085.019942:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776085.339912:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776085.339922:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776085.339925:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776085.339929:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776085.339933:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776085.339940:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17800 00000400:00000200:1.0:1775776085.339950:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776085.339956:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776085.339958:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776085.339961:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.339963:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.339965:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.339967:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776085.339970:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776085.339972:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776085.339974:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776085.339976:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776085.339978:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776085.339979:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776085.339980:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.339982:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.339983:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.339984:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776085.339987:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776085.339989:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776085.339990:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776085.339992:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776085.339995:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776085.339997:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.339999:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776085.340000:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776085.340010:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776085.340013:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776085.340016:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776085.340018:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340020:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776085.340022:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776085.340025:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776085.340033:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776085.340054:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776085.340065:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099706752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776085.340077:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776085.340083:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776085.340088:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099706752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776085.340111:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776085.340115:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776085.340119:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776085.340120:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776085.340122:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776085.340125:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776085.340127:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17800 00000400:00000200:1.0:1775776085.340131:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776085.340135:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776085.340137:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776085.340141:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340143:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.340146:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340147:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776085.340149:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776085.340150:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776085.340151:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776085.340153:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776085.340154:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776085.340156:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776085.340160:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340161:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.340164:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340166:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776085.340170:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776085.340175:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340177:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776085.340180:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776085.340185:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776085.340187:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340189:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776085.340194:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776085.340197:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340200:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776085.340202:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776085.340206:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776085.340209:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776085.340211:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340214:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776085.340215:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776085.340218:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776085.340223:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775776085.340232:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776085.340237:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099706880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776085.340244:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776085.340248:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776085.340251:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099706880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776085.340263:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776085.340265:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776085.340269:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776085.340270:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776085.340271:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776085.340274:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776085.340278:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17800 00000400:00000200:1.0:1775776085.340281:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776085.340283:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776085.340284:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776085.340289:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340290:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.340294:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340296:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776085.340298:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776085.340300:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776085.340301:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776085.340304:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776085.340307:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776085.340310:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776085.340316:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340319:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.340323:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340326:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776085.340328:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776085.340331:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340333:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776085.340335:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776085.340337:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776085.340341:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340344:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776085.340345:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776085.340347:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340348:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776085.340352:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776085.340355:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776085.340357:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776085.340358:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340359:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776085.340360:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776085.340362:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776085.340367:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776085.340370:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776085.340376:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099707008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776085.340385:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776085.340388:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776085.340394:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099707008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776085.340406:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776085.340407:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776085.340413:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776085.340416:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776085.340419:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776085.340422:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776085.340425:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17800 00000400:00000200:1.0:1775776085.340434:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776085.340437:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776085.340439:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776085.340442:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340443:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.340444:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340445:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776085.340449:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776085.340450:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776085.340452:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776085.340455:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776085.340457:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776085.340460:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776085.340464:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340465:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.340468:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776085.340468:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776085.340471:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776085.340474:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000800:00000200:2.1:1775776085.340474:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00080000:1.0:1775776085.340476:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340480:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776085.340482:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776085.340484:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776085.340486:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340488:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776085.340491:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776085.340492:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340495:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776085.340498:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776085.340499:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776085.340500:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776085.340501:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340502:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776085.340504:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776085.340506:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776085.340509:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776085.340512:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776085.340517:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099707136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776085.340524:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776085.340527:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776085.340530:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099707136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776085.340538:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776085.340541:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776085.340545:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776085.340546:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776085.340548:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776085.340553:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776085.340557:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17800 00000400:00000200:1.0:1775776085.340560:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776085.340562:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776085.340564:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776085.340568:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340571:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.340574:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340577:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776085.340581:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776085.340582:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776085.340585:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776085.340587:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776085.340588:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776085.340593:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776085.340596:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340598:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776085.340599:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340600:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776085.340602:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776085.340606:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776085.340608:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776085.340611:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776085.340613:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776085.340615:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340616:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776085.340619:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776085.340621:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340622:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776085.340624:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776085.340628:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776085.340629:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776085.340629:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776085.340630:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776085.340631:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776085.340632:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776085.340634:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776085.340636:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776085.340640:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099707264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776085.340644:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776085.340646:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776085.340650:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099707264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776085.340660:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776085.340666:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17810) 00000800:00000200:2.1:1775776085.340884:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776085.340899:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776085.341137:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776085.341138:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776085.341298:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776085.341300:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776085.341453:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776085.341455:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1775776086.043880:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776086.043888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776086.043890:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776086.043895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776086.043898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776086.043902:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776086.043905:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776086.043907:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776086.043909:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776086.619885:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312474357, at 4312474368 00000100:00000010:1.0:1775776086.619902:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000001:1.0:1775776087.067874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776087.067881:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776087.067882:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776087.067888:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776087.067891:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776087.067894:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776087.067896:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776087.067899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776087.067901:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776087.131974:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776087.131978:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776087.131982:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776088.091896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776088.091904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776088.091906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776088.091912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776088.091916:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776088.091920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776088.091922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776088.091924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776088.091927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776089.115881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776089.115886:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776089.115888:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776089.115891:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776089.115894:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776089.115897:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776089.115898:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776089.115900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776089.115902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776090.139896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776090.139903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776090.139904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776090.139909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776090.139912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776090.139916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776090.139918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776090.139920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776090.139922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776090.203986:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776090.203989:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776090.203993:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776090.460498:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776090.460503:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776090.460990:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776090.460992:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776090.461248:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776090.461251:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776090.461448:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776090.461450:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776090.461658:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776090.461661:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1775776091.164059:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776091.164072:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776091.164074:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776091.164083:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:1.0:1775776091.164089:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776091.164095:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776091.164098:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776091.164100:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776091.164104:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776091.739887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312479477, at 4312479488 00000100:00000010:1.0:1775776091.739907:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775776093.276023:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776093.276028:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776093.276031:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776095.580260:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776095.580264:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775776095.580849:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776095.580852:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775776095.581041:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776095.581042:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775776095.581191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776095.581193:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1775776095.581392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776095.581394:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000020:00000001:1.0:1775776096.348148:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776096.348151:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776096.348154:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776096.859879:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312484597, at 4312484608 00000100:00000010:1.0:1775776096.859891:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000020:00000001:1.0:1775776099.420063:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776099.420067:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776099.420071:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776100.700441:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776100.700446:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776100.701008:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776100.701011:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776100.701281:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776100.701284:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776100.701544:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776100.701558:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776100.701848:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776100.701851:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000100:00000040:1.0:1775776101.979917:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312489717, at 4312489728 00000100:00000010:1.0:1775776101.979934:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775776102.492005:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776102.492008:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776102.492010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776105.564924:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776105.564927:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776105.564930:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776105.820413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776105.820419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776105.820995:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776105.820997:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776105.821161:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776105.821162:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776105.821343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776105.821344:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776105.821542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776105.821544:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000100:00000040:1.0:1775776107.099881:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312494837, at 4312494848 00000100:00000010:1.0:1775776107.099895:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775776108.635939:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776108.635942:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776108.635945:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776110.940477:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776110.940924:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776110.941083:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776110.941251:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776110.941401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775776111.707967:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776111.707971:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776111.707974:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776112.219884:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312499957, at 4312499968 00000100:00000010:1.0:1775776112.219899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775776114.780021:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776114.780025:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776114.780028:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776116.060723:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776116.060726:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776116.061067:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776116.061069:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776116.061312:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776116.061315:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776116.061558:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776116.061572:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776116.061938:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776116.061939:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000040:1.0:1775776117.339874:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312505078, at 4312505088 00000100:00000010:1.0:1775776117.339889:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775776117.851983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776117.851986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776117.851988:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776120.923967:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776120.923971:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776120.923974:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776121.180372:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776121.180377:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776121.180994:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776121.180996:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776121.181221:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776121.181222:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776121.181444:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776121.181446:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776121.181650:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776121.181652:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1775776122.459870:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312510197, at 4312510208 00000100:00000010:1.0:1775776122.459879:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776123.995965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776123.995969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776123.995972:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776126.299901:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776126.299908:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776126.299910:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776126.299913:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776126.299916:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776126.299921:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17841 00000400:00000200:1.0:1775776126.299928:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776126.299932:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776126.299935:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776126.299937:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.299939:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.299941:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.299942:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776126.299945:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776126.299946:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776126.299948:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776126.299950:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776126.299951:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776126.299953:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776126.299954:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.299955:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.299957:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.299958:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776126.299960:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776126.299962:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776126.299963:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776126.299965:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776126.299967:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776126.299969:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.299970:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776126.299971:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776126.299978:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776126.299981:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776126.299983:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776126.299984:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.299986:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776126.299988:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776126.299991:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776126.299998:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776126.300004:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776126.300012:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099711872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776126.300027:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776126.300032:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776126.300037:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099711872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776126.300055:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776126.300058:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776126.300060:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776126.300061:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776126.300062:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776126.300064:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776126.300066:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17841 00000400:00000200:1.0:1775776126.300068:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776126.300070:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776126.300072:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776126.300074:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300075:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.300077:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300078:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776126.300080:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776126.300081:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776126.300082:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776126.300083:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776126.300084:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776126.300086:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776126.300088:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300090:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.300092:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300094:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776126.300097:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776126.300102:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300106:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776126.300108:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776126.300114:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776126.300119:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300122:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776126.300127:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776126.300133:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300137:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776126.300142:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776126.300146:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776126.300149:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776126.300152:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300154:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776126.300156:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776126.300158:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776126.300163:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776126.300167:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776126.300173:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099712000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776126.300182:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776126.300185:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776126.300189:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099712000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776126.300198:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776126.300200:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776126.300203:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776126.300206:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776126.300209:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776126.300212:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776126.300217:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17841 00000400:00000200:1.0:1775776126.300221:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776126.300226:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776126.300229:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776126.300233:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300236:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.300238:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300241:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776126.300243:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776126.300245:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776126.300248:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776126.300249:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776126.300251:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776126.300255:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776126.300258:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300260:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.300263:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300265:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776126.300266:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776126.300270:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300273:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776126.300275:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776126.300277:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776126.300278:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300280:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776126.300283:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776126.300285:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300287:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776126.300291:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776126.300294:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776126.300298:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776126.300301:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300306:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776126.300310:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776126.300313:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776126.300319:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776126.300324:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776126.300342:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099712128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776126.300355:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776126.300360:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776126.300367:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099712128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776126.300371:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776126.300376:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1775776126.300380:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776126.300383:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776126.300386:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776126.300388:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776126.300390:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776126.300394:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776126.300396:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17841 00000400:00000200:1.0:1775776126.300400:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776126.300404:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776126.300407:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776126.300412:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300414:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.300416:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300417:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776126.300421:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776126.300422:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776126.300423:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776126.300425:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776126.300428:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776126.300430:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776126.300433:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300437:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.300440:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300443:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776126.300446:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776126.300450:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300454:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776126.300457:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776126.300460:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776126.300462:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300463:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776126.300467:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776126.300471:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300473:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776126.300477:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776126.300480:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776126.300484:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776126.300485:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300487:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776126.300489:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776126.300491:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776126.300494:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776126.300497:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776126.300505:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099712256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776126.300511:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776126.300513:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776126.300515:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099712256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776126.300522:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776126.300524:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776126.300526:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776126.300526:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776126.300527:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776126.300528:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776126.300530:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17841 00000400:00000200:1.0:1775776126.300532:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776126.300535:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776126.300537:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776126.300541:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300545:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.300547:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300550:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776126.300553:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776126.300556:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776126.300557:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776126.300559:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776126.300562:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776126.300570:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776126.300575:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300578:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776126.300581:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300584:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776126.300587:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776126.300592:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776126.300596:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776126.300599:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776126.300602:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776126.300605:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300607:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776126.300610:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776126.300614:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300617:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776126.300621:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776126.300625:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776126.300629:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776126.300631:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776126.300634:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776126.300636:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776126.300639:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776126.300642:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776126.300646:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776126.300654:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099712384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776126.300668:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776126.300671:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776126.300675:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099712384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776126.300687:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776126.300691:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17851) 00000800:00000200:2.1:1775776126.300876:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776126.300879:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776126.301061:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776126.301062:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776126.301270:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776126.301272:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776126.301465:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776126.301467:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775776127.067983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776127.067987:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776127.067990:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776127.579889:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312515317, at 4312515328 00000100:00000010:1.0:1775776127.579899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775776130.139974:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776130.139977:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776130.139981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776131.163939:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776131.163943:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776131.163946:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776131.420347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776131.420362:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776131.420907:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776131.420910:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776131.421290:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776131.421293:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776131.421608:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776131.421610:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776131.421880:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776131.421881:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000040:1.0:1775776132.699882:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312520437, at 4312520448 00000100:00000010:1.0:1775776132.699888:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775776133.211943:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776133.211946:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776133.211948:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776134.235937:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776134.235940:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776134.235942:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776136.284025:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776136.284029:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776136.284031:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776136.540403:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776136.540407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776136.540902:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776136.540904:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776136.541140:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776136.541142:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776136.541380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776136.541381:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776136.541563:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776136.541565:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775776137.307955:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776137.307957:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776137.307960:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776137.819975:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312525557, at 4312525568 00000100:00000010:1.0:1775776137.819988:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775776139.356218:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776139.356221:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776139.356223:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776140.380016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776140.380020:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776140.380024:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776141.660043:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776141.660055:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776141.660058:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776141.660075:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776141.660080:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776141.660089:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17856 00000400:00000200:1.0:1775776141.660100:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776141.660108:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776141.660112:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776141.660115:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660119:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.660122:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660124:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776141.660130:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776141.660132:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776141.660135:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776141.660139:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776141.660142:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776141.660144:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776141.660147:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660149:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.660151:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660153:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776141.660156:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776141.660159:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660162:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776141.660165:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776141.660169:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776141.660172:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660175:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776141.660178:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776141.660191:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776141.660196:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776141.660201:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776141.660203:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660206:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776141.660209:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776141.660214:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776141.660225:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776141.660233:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776141.660243:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099713792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776141.660254:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776141.660260:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776141.660265:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099713792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776141.660292:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776141.660296:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776141.660299:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776141.660300:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776141.660302:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776141.660304:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776141.660306:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17856 00000400:00000200:1.0:1775776141.660310:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776141.660313:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776141.660316:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776141.660319:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660322:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.660325:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660327:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776141.660329:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776141.660331:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776141.660333:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776141.660336:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776141.660339:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776141.660344:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776141.660350:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660355:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.660359:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660364:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776141.660369:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776141.660375:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660379:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776141.660382:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776141.660390:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776141.660395:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660399:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776141.660406:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776141.660411:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660417:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776141.660424:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776141.660427:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776141.660432:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776141.660443:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660447:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776141.660449:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776141.660450:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776141.660456:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776141.660462:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776141.660471:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099713920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776141.660488:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776141.660493:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776141.660501:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099713920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776141.660516:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776141.660521:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776141.660528:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776141.660530:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776141.660532:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776141.660538:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776141.660544:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17856 00000400:00000200:1.0:1775776141.660551:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776141.660558:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776141.660563:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776141.660570:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660573:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.660578:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660584:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776141.660590:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776141.660592:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776141.660594:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776141.660596:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776141.660599:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776141.660603:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776141.660611:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660614:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.660619:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660623:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776141.660626:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776141.660630:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660633:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776141.660637:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776141.660641:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776141.660645:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660650:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776141.660652:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.1:1775776141.660653:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00000001:1.0:1775776141.660657:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776141.660659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:1.0:1775776141.660661:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776141.660667:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776141.660673:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776141.660678:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776141.660681:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660682:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776141.660684:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776141.660687:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776141.660693:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776141.660699:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776141.660708:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099714048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776141.660720:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776141.660724:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776141.660732:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099714048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776141.660745:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776141.660749:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776141.660757:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776141.660759:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776141.660762:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776141.660766:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776141.660770:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17856 00000400:00000200:1.0:1775776141.660777:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776141.660831:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776141.660836:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776141.660840:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660844:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.660849:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660853:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776141.660857:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776141.660860:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776141.660863:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776141.660866:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776141.660871:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776141.660875:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776141.660880:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660884:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.660888:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660893:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776141.660898:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776141.660902:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776141.660907:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776141.660910:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776141.660915:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776141.660919:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660923:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776141.660926:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776141.660933:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660937:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776141.660944:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776141.660949:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776141.660954:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776141.660957:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.660961:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776141.660964:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776141.660967:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776141.660973:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776141.660979:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776141.660988:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099714176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776141.661005:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776141.661009:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776141.661017:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099714176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776141.661035:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776141.661040:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776141.661048:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776141.661051:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776141.661055:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776141.661072:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776141.661077:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17856 00000400:00000200:1.0:1775776141.661084:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776141.661089:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776141.661095:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776141.661099:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.661104:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.661107:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.661111:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776141.661115:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776141.661118:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776141.661120:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776141.661124:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776141.661129:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776141.661133:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776141.661136:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.661140:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776141.661144:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776141.661148:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776141.661153:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776141.661160:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776141.661166:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776141.661170:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776141.661174:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776141.661180:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.661182:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776141.661184:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776141.661187:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.661191:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776141.661198:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776141.661202:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776141.661209:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776141.661212:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776141.661215:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776141.661218:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776141.661221:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776141.661227:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776141.661233:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776141.661241:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099714304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776141.661252:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776141.661255:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776141.661263:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099714304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776141.661275:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776141.661283:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17867) 00000800:00000200:2.1:1775776141.661337:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776141.661339:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776141.661563:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776141.661579:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776141.661814:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776141.661817:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776141.662223:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776141.662226:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775776142.427995:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776142.427999:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776142.428003:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776142.939925:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312530678, at 4312530688 00000100:00000010:1.0:1775776142.939934:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775776143.451981:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776143.451985:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776143.451989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776145.500049:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776145.500053:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776145.500056:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776146.524024:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776146.524027:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776146.524029:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776146.780546:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776146.780549:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776146.781165:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776146.781167:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776146.781430:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776146.781432:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776146.781695:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776146.781697:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776146.781984:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776146.781987:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000040:1.0:1775776148.059907:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312535797, at 4312535808 00000100:00000010:1.0:1775776148.059916:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775776148.572017:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776148.572020:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776148.572022:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776149.595995:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776149.595999:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776149.596001:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776151.643989:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776151.643991:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776151.643993:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776151.899915:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776151.899923:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776151.899926:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776151.899930:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776151.899935:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776151.899941:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17867 00000400:00000200:1.0:1775776151.899950:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776151.899956:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776151.899959:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776151.899961:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.899964:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.899966:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.899968:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776151.899971:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776151.899973:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776151.899975:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776151.899978:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776151.899980:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776151.899982:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776151.899984:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.899986:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.899987:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.899989:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776151.899992:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776151.899994:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776151.899997:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776151.899999:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776151.900002:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776151.900004:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900006:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776151.900008:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776151.900016:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776151.900020:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776151.900023:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776151.900025:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900028:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776151.900030:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776151.900034:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776151.900041:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776151.900046:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776151.900054:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099715072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776151.900062:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776151.900065:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776151.900068:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099715072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776151.900084:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776151.900087:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776151.900090:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776151.900091:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776151.900093:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776151.900095:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776151.900097:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17867 00000400:00000200:1.0:1775776151.900102:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776151.900106:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776151.900109:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776151.900112:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900115:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.900120:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900124:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776151.900128:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776151.900130:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776151.900134:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776151.900138:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776151.900141:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776151.900144:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776151.900150:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900153:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.900156:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900158:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776151.900161:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776151.900165:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900167:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776151.900177:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776151.900181:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776151.900183:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900184:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776151.900186:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776151.900189:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900192:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776151.900198:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776151.900200:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776151.900204:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776151.900206:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900209:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776151.900210:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776151.900212:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776151.900222:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776151.900226:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776151.900230:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099715200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776151.900242:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776151.900245:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776151.900249:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099715200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776151.900270:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776151.900275:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776151.900282:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776151.900284:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776151.900286:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776151.900290:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776151.900300:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17867 00000400:00000200:1.0:1775776151.900308:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776151.900313:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776151.900318:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776151.900323:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900326:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.900331:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900334:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776151.900338:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776151.900341:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776151.900344:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776151.900347:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776151.900356:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776151.900361:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776151.900368:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900371:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.900376:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900379:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776151.900381:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776151.900383:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900384:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776151.900386:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776151.900388:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776151.900390:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900392:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776151.900394:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776151.900396:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900397:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776151.900401:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776151.900404:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776151.900405:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776151.900406:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900408:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776151.900410:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776151.900413:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776151.900417:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776151.900423:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776151.900429:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00100000:1.0:1775776151.900432:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099715328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775776151.900433:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1775776151.900449:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776151.900452:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776151.900460:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099715328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776151.900482:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776151.900486:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776151.900492:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776151.900494:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776151.900496:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776151.900500:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776151.900504:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17867 00000400:00000200:1.0:1775776151.900509:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776151.900512:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776151.900514:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776151.900518:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900521:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.900525:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900530:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776151.900535:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776151.900537:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776151.900540:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776151.900543:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776151.900546:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776151.900549:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776151.900556:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900559:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.900564:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900569:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776151.900572:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776151.900585:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900588:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776151.900590:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776151.900595:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776151.900599:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900603:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776151.900605:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776151.900609:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900611:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776151.900616:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776151.900618:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776151.900620:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776151.900621:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900622:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776151.900623:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776151.900624:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776151.900628:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776151.900632:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776151.900641:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099715456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776151.900650:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776151.900654:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776151.900663:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099715456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776151.900684:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776151.900693:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776151.900699:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776151.900702:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776151.900704:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776151.900709:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776151.900714:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17867 00000400:00000200:1.0:1775776151.900722:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776151.900726:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776151.900729:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776151.900733:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900736:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.900740:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900744:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776151.900748:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776151.900749:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776151.900757:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776151.900760:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776151.900763:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776151.900765:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776151.900771:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900775:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776151.900811:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900816:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776151.900819:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776151.900825:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776151.900829:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776151.900832:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776151.900835:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776151.900840:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900844:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776151.900847:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776151.900852:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900855:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776151.900861:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776151.900863:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776151.900866:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776151.900869:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776151.900870:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776151.900873:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776151.900877:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776151.900893:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776151.900896:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776151.900903:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099715584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776151.900919:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776151.900923:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776151.900928:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099715584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776151.900940:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776151.900946:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17877) 00000800:00000200:2.1:1775776151.901040:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776151.901042:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776151.901261:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776151.901262:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776151.901476:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776151.901478:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776151.901731:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776151.901745:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775776152.667956:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776152.667959:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776152.667962:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776153.179894:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312540917, at 4312540928 00000100:00000010:1.0:1775776153.179900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775776154.715971:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776154.715974:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776154.715977:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776155.739964:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776155.739967:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776155.739969:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776157.020359:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776157.020363:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776157.020973:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776157.020976:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776157.021205:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776157.021207:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776157.021404:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776157.021406:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776157.021689:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776157.021691:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775776157.787976:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776157.787980:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776157.787983:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776158.299888:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312546037, at 4312546048 00000100:00000010:1.0:1775776158.299903:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776158.812016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776158.812020:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776158.812022:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776160.859989:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776160.859992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776160.859996:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776161.883954:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776161.883957:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776161.883959:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776162.140438:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776162.140444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776162.141074:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776162.141077:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776162.141343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776162.141345:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776162.141607:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776162.141610:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776162.141908:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776162.141911:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000040:1.0:1775776163.419871:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312551157, at 4312551168 00000100:00000010:1.0:1775776163.419878:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775776163.931965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776163.931968:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776163.931971:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776164.955925:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776164.955929:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776164.955931:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776167.003952:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776167.003955:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776167.003958:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776167.259885:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776167.259893:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776167.259895:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776167.259898:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776167.259901:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776167.259906:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17882 00000400:00000200:1.0:1775776167.259914:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776167.259919:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776167.259921:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776167.259923:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.259926:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.259927:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.259929:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776167.259932:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776167.259933:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776167.259935:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776167.259937:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776167.259938:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776167.259939:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776167.259941:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.259942:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.259944:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.259945:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776167.259947:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776167.259950:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776167.259957:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776167.259960:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776167.259963:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776167.259965:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.259967:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776167.259969:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776167.259977:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776167.259981:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776167.259985:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776167.259987:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.259990:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776167.259992:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776167.259996:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776167.260003:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775776167.260008:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776167.260015:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099716992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776167.260022:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776167.260025:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776167.260028:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099716992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776167.260043:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776167.260045:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776167.260047:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776167.260048:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776167.260049:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776167.260051:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776167.260053:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17882 00000400:00000200:1.0:1775776167.260055:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776167.260057:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776167.260059:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776167.260060:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260061:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.260063:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260064:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776167.260065:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776167.260066:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776167.260068:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776167.260069:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776167.260070:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776167.260071:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776167.260074:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260075:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.260082:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260084:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776167.260085:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776167.260090:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260094:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776167.260095:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776167.260100:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776167.260102:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260103:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776167.260107:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776167.260112:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260116:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776167.260122:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776167.260125:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776167.260129:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776167.260132:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260134:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776167.260137:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776167.260139:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776167.260144:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776167.260149:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776167.260157:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099717120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776167.260170:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776167.260173:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776167.260175:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099717120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776167.260186:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776167.260188:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776167.260202:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776167.260205:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776167.260207:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776167.260210:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776167.260213:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17882 00000400:00000200:1.0:1775776167.260221:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776167.260225:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776167.260228:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776167.260234:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260237:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.260241:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260244:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776167.260248:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776167.260250:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776167.260252:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776167.260255:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776167.260259:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776167.260263:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776167.260270:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260273:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.260277:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260280:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776167.260284:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776167.260289:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260294:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776167.260296:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776167.260299:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776167.260303:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260313:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776167.260317:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776167.260320:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260323:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776167.260326:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776167.260328:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776167.260331:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776167.260333:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260337:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776167.260338:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776167.260341:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776167.260348:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776167.260354:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776167.260362:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099717248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776167.260377:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776167.260389:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776167.260396:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099717248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776167.260404:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776167.260408:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776167.260412:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776167.260414:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776167.260415:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776167.260417:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:2.1:1775776167.260420:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:1.0:1775776167.260421:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17882 00000400:00000200:1.0:1775776167.260426:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000800:00000200:2.1:1775776167.260426:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000400:00000200:1.0:1775776167.260430:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776167.260432:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776167.260437:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260439:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.260442:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260444:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776167.260446:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776167.260447:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776167.260448:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776167.260449:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776167.260451:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776167.260455:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776167.260457:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260460:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.260467:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260469:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776167.260472:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776167.260475:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260478:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776167.260479:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776167.260481:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776167.260484:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260485:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776167.260488:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776167.260492:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260496:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776167.260500:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776167.260503:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776167.260506:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776167.260508:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260509:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776167.260510:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776167.260513:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776167.260516:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776167.260519:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776167.260526:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099717376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776167.260536:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776167.260538:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776167.260541:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099717376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776167.260546:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776167.260550:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776167.260555:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776167.260557:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776167.260559:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776167.260560:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776167.260564:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17882 00000400:00000200:1.0:1775776167.260566:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776167.260568:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776167.260572:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776167.260574:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260575:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.260577:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260581:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776167.260582:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776167.260583:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776167.260584:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776167.260586:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776167.260587:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776167.260591:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776167.260593:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260595:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776167.260598:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260599:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776167.260601:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776167.260611:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776167.260633:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776167.260636:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776167.260639:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776167.260642:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260645:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776167.260648:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776167.260652:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260653:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776167.260655:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776167.260658:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776167.260661:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776167.260662:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776167.260665:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776167.260667:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776167.260669:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776167.260672:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776167.260676:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776167.260681:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099717504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776167.260708:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776167.260710:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776167.260714:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099717504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776167.260741:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776167.260747:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17892) 00000800:00000200:2.1:1775776167.261013:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776167.261027:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776167.261302:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776167.261305:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776167.261565:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776167.261567:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776167.261862:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776167.261865:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775776168.027990:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776168.028007:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776168.028010:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776168.539886:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312556277, at 4312556288 00000100:00000010:1.0:1775776168.539892:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775776170.075917:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776170.075921:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776170.075923:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776171.099991:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776171.099994:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776171.099997:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776172.380578:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776172.380584:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776172.381198:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776172.381200:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776172.381405:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776172.381407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776172.381606:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776172.381608:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776172.381796:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776172.381798:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775776173.147996:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776173.148000:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776173.148005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776173.659883:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312561397, at 4312561408 00000100:00000010:1.0:1775776173.659891:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775776174.171930:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776174.171934:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776174.171937:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776176.219998:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776176.220001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776176.220003:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776177.243972:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776177.243976:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776177.243980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776177.500495:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776177.500981:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776177.501194:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776177.501344:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776177.501498:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775776178.268022:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776178.268026:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776178.268029:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776178.779918:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312566517, at 4312566528 00000100:00000010:1.0:1775776178.779940:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775776179.292055:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776179.292100:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776179.292103:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776180.316007:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776180.316011:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776180.316013:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775776180.572042:0:301877:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775776180.572101:0:301877:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775776180.572112:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776180.572116:0:301877:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775776180.572120:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776180.572124:0:301877:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775776180.572128:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776180.572144:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7c00. 00000400:00000200:1.0:1775776180.572161:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776180.572165:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776180.572171:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d908 00000400:00000010:1.0:1775776180.572173:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d908. 00000100:00000001:1.0:1775776180.572191:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776180.572206:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45ded180 x1862033099718272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572235:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776180.572239:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099718272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572249:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.572251:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776180.572260:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776180.572262:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:1.0:1775776180.572265:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776180.572268:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776180.572270:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d3b8 00000400:00000010:1.0:1775776180.572272:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d3b8. 00000100:00000001:1.0:1775776180.572274:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776180.572277:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dedf80 x1862033099718400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572284:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776180.572286:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedf80 x1862033099718400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572291:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.572292:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776180.572296:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776180.572298:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7000. 00000400:00000200:1.0:1775776180.572300:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776180.572302:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776180.572305:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dee0 00000400:00000010:1.0:1775776180.572306:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dee0. 00000100:00000001:1.0:1775776180.572308:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776180.572312:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45deea00 x1862033099718528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572318:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776180.572320:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099718528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572326:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.572327:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776180.572331:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776180.572333:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7700. 00000400:00000200:1.0:1775776180.572336:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776180.572338:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776180.572341:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d660 00000400:00000010:1.0:1775776180.572343:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d660. 00000100:00000001:1.0:1775776180.572345:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776180.572348:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45def800 x1862033099718656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572355:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776180.572357:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def800 x1862033099718656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572362:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.572364:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776180.572367:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776180.572369:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:1.0:1775776180.572372:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776180.572374:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776180.572377:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d770 00000400:00000010:1.0:1775776180.572378:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d770. 00000100:00000001:1.0:1775776180.572380:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776180.572414:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dec000 x1862033099718784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572420:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776180.572423:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099718784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.572428:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.572430:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775776181.339950:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776181.339952:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776181.339954:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776182.363968:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776182.363971:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776182.363973:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776182.620192:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776182.620203:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776182.620206:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776182.620210:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776182.620215:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776182.620222:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17897 00000400:00000200:1.0:1775776182.620232:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776182.620238:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776182.620241:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776182.620244:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620247:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620249:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620252:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776182.620256:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776182.620259:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776182.620261:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776182.620296:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776182.620300:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776182.620303:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776182.620306:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620308:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620310:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620312:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776182.620316:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776182.620318:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620321:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776182.620325:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776182.620329:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776182.620332:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620334:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776182.620337:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776182.620344:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776182.620379:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776182.620384:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776182.620386:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620390:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776182.620393:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776182.620397:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776182.620405:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776182.620411:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776182.620420:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099718912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776182.620431:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776182.620436:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776182.620441:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099718912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776182.620477:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776182.620480:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776182.620484:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776182.620486:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776182.620487:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776182.620490:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776182.620493:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17897 00000400:00000200:1.0:1775776182.620497:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776182.620500:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776182.620555:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776182.620560:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620563:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620564:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620567:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776182.620569:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776182.620571:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776182.620573:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776182.620574:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776182.620577:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776182.620579:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776182.620582:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620584:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620586:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620588:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776182.620590:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776182.620592:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620594:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776182.620596:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776182.620601:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776182.620603:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620605:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776182.620609:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776182.620612:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620614:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776182.620618:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776182.620620:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776182.620622:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776182.620624:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620625:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776182.620626:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776182.620628:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776182.620632:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776182.620635:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776182.620640:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099719040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776182.620648:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776182.620650:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776182.620654:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099719040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776182.620661:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776182.620664:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776182.620668:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776182.620669:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776182.620670:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776182.620673:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776182.620675:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17897 00000400:00000200:1.0:1775776182.620679:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776182.620682:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776182.620684:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776182.620687:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620689:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620691:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620693:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776182.620695:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776182.620696:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776182.620705:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776182.620707:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776182.620709:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776182.620712:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776182.620715:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620717:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620718:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620720:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776182.620722:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776182.620725:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620727:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776182.620729:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776182.620732:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776182.620734:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620736:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776182.620739:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776182.620741:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620743:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776182.620746:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776182.620748:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776182.620750:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776182.620751:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620753:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776182.620754:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776182.620756:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776182.620759:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776182.620762:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776182.620805:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099719168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776182.620812:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776182.620814:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776182.620818:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099719168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776182.620825:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776182.620827:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776182.620830:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776182.620831:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776182.620833:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776182.620835:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776182.620837:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17897 00000400:00000200:1.0:1775776182.620845:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776182.620848:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776182.620850:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776182.620853:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620855:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620857:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620859:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776182.620862:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776182.620863:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776182.620865:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776182.620867:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776182.620869:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776182.620871:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776182.620873:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620875:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620877:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620879:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776182.620881:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776182.620884:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620886:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776182.620888:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776182.620891:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776182.620893:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620895:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776182.620897:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776182.620899:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620901:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776182.620905:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776182.620907:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776182.620909:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776182.620910:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.620912:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776182.620913:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776182.620915:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776182.620918:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776182.620921:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776182.620926:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099719296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776182.620938:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776182.620940:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776182.620944:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099719296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776182.620951:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776182.620954:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776182.620957:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776182.620958:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776182.620960:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776182.620962:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776182.620965:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17897 00000400:00000200:1.0:1775776182.620969:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776182.620972:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776182.620974:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776182.620976:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620978:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620980:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620982:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776182.620984:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776182.620986:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776182.620987:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776182.620989:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776182.620991:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776182.620993:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776182.620995:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.620997:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776182.620999:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776182.621001:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776182.621003:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776182.621006:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776182.621008:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776182.621010:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776182.621012:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776182.621014:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.621016:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776182.621018:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776182.621021:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.621023:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776182.621026:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776182.621028:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776182.621030:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776182.621032:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776182.621034:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776182.621035:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776182.621037:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776182.621041:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776182.621044:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776182.621048:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099719424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776182.621055:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776182.621057:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776182.621060:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099719424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776182.621067:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776182.621072:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17908) 00000800:00000200:2.1:1775776182.622057:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776182.622059:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776182.622385:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776182.622419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776182.622676:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776182.622678:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776182.623625:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776182.623628:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776182.623888:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776182.623889:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775776183.387980:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776183.387984:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776183.387988:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776183.899942:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312571639, at 4312571648 00000100:00000010:1.0:1775776183.899960:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775776184.412035:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776184.412040:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776184.412043:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776185.435980:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776185.436024:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776185.436029:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776186.460065:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776186.460070:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776186.460074:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776187.484050:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776187.484053:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776187.484056:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776187.740576:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776187.740582:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776187.741166:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776187.741168:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776187.741346:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776187.741348:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776187.741568:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776187.741570:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776187.741799:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776187.741801:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775776188.507963:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776188.507967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776188.507970:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776189.019921:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312576757, at 4312576768 00000100:00000010:1.0:1775776189.019939:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775776189.531998:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776189.532002:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776189.532005:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776190.557129:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776190.558241:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776190.558245:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776191.580020:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776191.580024:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776191.580027:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776192.603999:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776192.604002:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776192.604005:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776192.860524:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776192.860537:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776192.861156:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776192.861158:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776192.861425:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776192.861427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776192.861677:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776192.861692:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776192.861961:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776192.861963:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775776193.627946:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776193.627950:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776193.627953:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776194.139910:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312581877, at 4312581888 00000100:00000010:1.0:1775776194.139919:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775776194.651997:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776194.652002:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776194.652005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776195.676038:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776195.676042:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776195.676045:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776196.699968:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776196.699970:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776196.699973:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776197.724008:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776197.724011:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776197.724013:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776197.980428:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776197.980434:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776197.981076:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776197.981090:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776197.981338:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776197.981340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776197.981588:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776197.981590:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776197.981850:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776197.981852:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775776198.748003:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776198.748007:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776198.748009:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776199.259897:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312586997, at 4312587008 00000100:00000010:1.0:1775776199.259921:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775776199.772015:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776199.772018:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776199.772022:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776200.795954:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776200.795958:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776200.795963:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776201.819947:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776201.819951:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776201.819955:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776202.843983:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776202.843987:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776202.843990:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776203.100478:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776203.100483:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776203.101115:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776203.101118:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776203.101383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776203.101386:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776203.101638:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776203.101640:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776203.101907:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776203.101909:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775776203.867937:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776203.867941:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776203.867944:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776204.379914:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312592117, at 4312592128 00000100:00000010:1.0:1775776204.379929:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776204.892111:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776204.892116:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776204.892121:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776205.915977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776205.915982:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776205.915985:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776206.940066:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776206.940070:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776206.940073:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776207.964075:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776207.964080:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776207.964084:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776208.220442:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776208.220460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776208.220949:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776208.220950:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776208.221139:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776208.221140:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776208.221303:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776208.221305:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776208.221506:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776208.221509:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000020:00000001:1.0:1775776208.988058:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776208.988062:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776208.988066:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776209.499962:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312597237, at 4312597248 00000100:00000010:1.0:1775776209.499971:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775776210.012041:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776210.012045:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776210.012047:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776211.036015:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776211.036019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776211.036023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776212.060019:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776212.060023:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776212.060026:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776213.083957:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776213.083960:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776213.083963:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776213.340294:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776213.340297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776213.340768:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776213.340770:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776213.340970:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776213.340971:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776213.341120:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776213.341122:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776213.341323:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776213.341324:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1775776214.107979:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776214.107983:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776214.107986:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776214.620015:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312602357, at 4312602368 00000100:00000010:1.0:1775776214.620019:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775776215.131954:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776215.131957:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776215.131959:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776216.155990:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776216.155993:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776216.156004:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776217.179939:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776217.179941:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776217.179944:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776218.203985:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776218.203988:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776218.203990:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776218.459906:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776218.459914:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776218.459916:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776218.459918:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776218.459921:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776218.459927:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17933 00000400:00000200:1.0:1775776218.459934:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776218.459938:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776218.459941:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776218.459943:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.459945:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.459947:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.459949:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776218.459951:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776218.459952:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776218.459955:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776218.459957:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776218.459959:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776218.459961:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776218.459964:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.459966:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.459967:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.459969:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776218.459973:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776218.459976:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776218.459978:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776218.459980:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776218.459983:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776218.459986:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.459988:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776218.459990:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776218.459999:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776218.460003:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776218.460007:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776218.460009:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460012:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776218.460015:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776218.460018:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776218.460025:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776218.460030:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776218.460038:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099723392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776218.460046:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776218.460049:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776218.460052:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099723392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776218.460070:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776218.460072:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776218.460074:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776218.460075:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776218.460076:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776218.460078:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776218.460080:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17933 00000400:00000200:1.0:1775776218.460083:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776218.460085:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776218.460087:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776218.460088:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460090:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.460091:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460092:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776218.460094:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776218.460095:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776218.460096:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776218.460098:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776218.460099:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776218.460101:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776218.460102:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460104:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.460105:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460106:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776218.460108:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776218.460112:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460113:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776218.460115:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776218.460121:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776218.460126:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460131:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776218.460136:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776218.460141:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460145:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776218.460152:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776218.460157:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776218.460161:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776218.460164:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460168:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776218.460171:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776218.460174:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776218.460181:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776218.460185:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776218.460193:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099723520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776218.460208:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776218.460212:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776218.460222:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099723520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776218.460240:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776218.460242:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776218.460248:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776218.460250:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776218.460251:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776218.460254:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776218.460257:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17933 00000400:00000200:1.0:1775776218.460264:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776218.460268:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776218.460272:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776218.460277:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460281:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.460285:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460288:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776218.460291:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776218.460293:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776218.460294:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776218.460296:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776218.460299:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776218.460304:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776218.460311:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460314:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.460318:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460319:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776218.460323:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776218.460325:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460329:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776218.460333:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776218.460336:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776218.460338:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776218.460340:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776218.460341:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776218.460344:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.1:1775776218.460344:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000008:00000001:1.0:1775776218.460347:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460350:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776218.460354:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776218.460356:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776218.460358:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776218.460360:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460363:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776218.460365:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776218.460367:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776218.460370:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776218.460374:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776218.460379:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099723648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776218.460383:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776218.460385:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776218.460390:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099723648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776218.460413:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776218.460417:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776218.460424:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776218.460427:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776218.460430:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776218.460435:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776218.460441:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17933 00000400:00000200:1.0:1775776218.460450:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776218.460456:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776218.460459:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776218.460462:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460467:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.460469:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460472:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776218.460476:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776218.460478:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776218.460480:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776218.460482:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776218.460486:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776218.460491:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776218.460498:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460500:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.460505:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460509:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776218.460513:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776218.460519:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460524:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776218.460527:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776218.460532:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776218.460535:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460539:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776218.460543:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776218.460547:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460550:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776218.460556:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776218.460561:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776218.460567:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776218.460570:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460575:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776218.460578:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776218.460583:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776218.460589:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776218.460595:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776218.460604:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099723776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776218.460625:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776218.460629:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776218.460636:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099723776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776218.460647:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776218.460650:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776218.460654:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776218.460658:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776218.460660:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776218.460663:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776218.460666:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17933 00000400:00000200:1.0:1775776218.460672:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776218.460678:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776218.460680:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776218.460684:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460688:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.460690:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460694:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776218.460698:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776218.460701:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776218.460703:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776218.460705:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776218.460708:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776218.460712:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776218.460716:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460719:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776218.460734:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460736:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776218.460739:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776218.460743:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776218.460746:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776218.460750:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776218.460752:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776218.460756:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460759:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776218.460762:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776218.460769:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460772:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776218.460814:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776218.460817:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776218.460822:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776218.460825:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776218.460826:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776218.460827:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776218.460829:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776218.460832:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776218.460835:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776218.460841:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099723904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776218.460863:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776218.460867:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776218.460873:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099723904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776218.460888:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776218.460893:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17943) 00000800:00000200:2.1:1775776218.460996:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776218.460999:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775776218.461190:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776218.461201:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775776218.461361:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776218.461362:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1775776218.461601:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776218.461603:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000020:00000001:1.0:1775776219.227984:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776219.227988:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776219.227991:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776219.739897:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312607477, at 4312607488 00000100:00000010:1.0:1775776219.739913:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775776220.251967:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776220.251970:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776220.251972:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776221.275936:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776221.275939:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776221.275941:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776222.299938:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776222.299942:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776222.299945:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776223.323920:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776223.323923:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776223.323927:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776223.580228:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776223.580232:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776223.580568:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776223.580569:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776223.580772:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776223.580774:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776223.580961:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776223.580963:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776223.581108:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776223.581109:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775776224.348906:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776224.348910:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776224.348913:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776224.859887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312612597, at 4312612608 00000100:00000010:1.0:1775776224.859895:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503447e0. 00000020:00000001:1.0:1775776225.371954:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776225.371958:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776225.371961:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776226.395975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776226.395977:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776226.395980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776227.419973:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776227.419977:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776227.419980:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776228.445053:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776228.445056:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776228.445058:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776228.700352:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776228.700357:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776228.700887:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776228.700889:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776228.701179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776228.701181:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776228.701434:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776228.701437:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776228.701726:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776228.701729:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775776229.468049:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776229.468053:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776229.468056:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776229.979866:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312617717, at 4312617728 00000100:00000010:1.0:1775776229.979875:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775776230.491924:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776230.491927:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776230.491929:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776231.515985:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776231.515989:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776231.515992:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776232.539946:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776232.539949:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776232.539952:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776233.563943:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776233.563955:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776233.563958:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776233.820391:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776233.820397:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000040:0.0:1775776233.820844:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776233.820848:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776233.820853:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776233.820858:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776233.820863:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776233.820871:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776233.820875:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776233.820880:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776233.820885:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776233.820889:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776233.820895:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776233.820899:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776233.820902:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776233.820906:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776233.820910:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776233.820913:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776233.820917:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776233.820922:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776233.820925:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776233.820931:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776233.820935:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776233.820940:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776233.820943:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776233.820947:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776233.820950:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776233.820953:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776233.820958:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776233.820965:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776233.820976:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099725824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776233.820996:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776233.820998:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776233.821005:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099725824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776233.821015:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776233.821019:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17959) 00000800:00000200:2.1:1775776233.821019:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776233.821024:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776233.821238:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776233.821240:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776233.821393:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776233.821394:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776233.821651:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776233.821662:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775776234.587930:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776234.587933:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776234.587936:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776235.099860:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312622837, at 4312622848 00000100:00000040:1.0:1775776235.099869:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312622837, at 4312622848 00000100:00000010:0.0:1775776235.099872:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000010:1.0:1775776235.099876:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775776235.611973:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776235.611976:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776235.611979:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776236.635942:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776236.635945:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776236.635948:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776237.659954:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776237.659957:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776237.659959:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776238.684003:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776238.684007:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776238.684010:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776238.940401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776238.940407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776238.940859:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776238.940861:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776238.941119:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776238.941120:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776238.941286:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776238.941287:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776238.941507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776238.941508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775776239.707932:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776239.707936:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776239.707939:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776240.219904:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312627957, at 4312627968 00000100:00000010:0.0:1775776240.219918:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000040:1.0:1775776240.219921:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312627957, at 4312627968 00000100:00000010:1.0:1775776240.219931:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775776240.732000:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776240.732004:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776240.732007:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776241.756000:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776241.756004:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776241.756007:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776242.779926:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776242.779929:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776242.779931:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776243.804063:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776243.804067:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776243.804070:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776244.060494:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776244.060845:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776244.061099:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776244.061242:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776244.061387:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775776244.827961:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776244.827965:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776244.827968:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776245.339872:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312633077, at 4312633088 00000100:00000040:1.0:1775776245.339886:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312633077, at 4312633088 00000100:00000010:0.0:1775776245.339894:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503440c0. 00000100:00000010:1.0:1775776245.339895:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000020:00000001:1.0:1775776245.852024:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776245.852027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776245.852029:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776246.876010:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776246.876013:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776246.876017:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775776247.132045:0:301878:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:0.0:1775776247.132084:0:301878:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775776247.132104:0:301878:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776247.132109:0:301878:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:0.0:1775776247.132113:0:301878:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776247.132118:0:301878:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775776247.132123:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776247.132126:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:0.0:1775776247.132131:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776247.132136:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776247.132142:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1198 00000400:00000010:0.0:1775776247.132145:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1198. 00000100:00000001:0.0:1775776247.132149:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776247.132157:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7c700 x1862033099726592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132188:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776247.132192:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c700 x1862033099726592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132200:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.132202:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776247.132211:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776247.132214:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511500. 00000400:00000200:0.0:1775776247.132217:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776247.132233:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776247.132236:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1d48 00000400:00000010:0.0:1775776247.132238:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1d48. 00000100:00000001:0.0:1775776247.132241:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776247.132245:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7d880 x1862033099726720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132252:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776247.132255:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099726720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132261:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.132263:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776247.132267:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776247.132270:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511400. 00000400:00000200:0.0:1775776247.132272:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776247.132275:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776247.132278:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1220 00000400:00000010:0.0:1775776247.132279:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1220. 00000100:00000001:0.0:1775776247.132282:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776247.132286:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7dc00 x1862033099726848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132293:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776247.132295:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099726848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132301:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.132303:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776247.132307:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776247.132309:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511f00. 00000400:00000200:0.0:1775776247.132311:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776247.132314:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776247.132317:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1000 00000400:00000010:0.0:1775776247.132319:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1000. 00000100:00000001:0.0:1775776247.132321:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776247.132324:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7f100 x1862033099726976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132332:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776247.132334:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f100 x1862033099726976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132341:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.132343:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776247.132347:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776247.132349:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511700. 00000400:00000200:0.0:1775776247.132352:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776247.132354:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776247.132357:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1aa0 00000400:00000010:0.0:1775776247.132358:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1aa0. 00000100:00000001:0.0:1775776247.132360:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776247.132364:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7f480 x1862033099727104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132371:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776247.132373:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099727104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.132379:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.132380:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775776247.899976:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776247.899980:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776247.899983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776248.924045:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776248.924048:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776248.924051:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776249.180721:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776249.180724:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776249.183265:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776249.183268:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776249.183675:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776249.183677:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776249.183940:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776249.183942:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776249.184178:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776249.184180:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775776249.947933:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776249.947936:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776249.947939:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776250.459872:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312638198, at 4312638208 00000100:00000010:0.0:1775776250.459884:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000040:1.0:1775776250.459887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312638198, at 4312638208 00000100:00000010:1.0:1775776250.459894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775776250.972021:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776250.972025:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776250.972028:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776251.995922:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776251.995924:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776251.995927:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776253.019962:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776253.019967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776253.019971:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776254.043958:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776254.043961:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776254.043964:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776254.300530:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776254.300536:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776254.301420:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776254.301437:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776254.301926:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776254.301927:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776254.302142:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776254.302144:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776254.302297:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776254.302299:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775776255.068005:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776255.068009:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776255.068012:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776255.579894:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312643317, at 4312643328 00000100:00000010:0.0:1775776255.579922:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000040:1.0:1775776255.579952:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312643317, at 4312643328 00000100:00000010:1.0:1775776255.579964:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775776256.092101:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776256.092105:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776256.092108:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776257.115993:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776257.115996:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776257.116000:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776258.139995:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776258.140001:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776258.140005:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776259.164099:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776259.164102:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776259.164105:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776259.419927:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776259.419937:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776259.419941:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776259.419946:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776259.419951:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776259.419960:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17974 00000400:00000200:0.0:1775776259.419974:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776259.419981:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776259.419985:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776259.419988:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.419992:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.419995:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.419998:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776259.420003:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776259.420005:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776259.420023:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776259.420026:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776259.420029:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776259.420031:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776259.420035:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420037:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420039:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420041:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776259.420044:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776259.420047:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420050:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776259.420054:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776259.420058:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776259.420061:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420064:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776259.420067:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776259.420080:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776259.420084:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776259.420089:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776259.420091:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420095:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776259.420098:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776259.420102:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776259.420110:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776259.420116:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776259.420125:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099728512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776259.420134:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776259.420138:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776259.420141:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099728512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776259.420163:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776259.420166:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776259.420170:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776259.420171:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776259.420173:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776259.420176:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776259.420178:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17974 00000400:00000200:0.0:1775776259.420182:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776259.420185:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776259.420187:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776259.420190:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420192:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420194:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420196:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776259.420198:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776259.420199:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776259.420201:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776259.420202:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776259.420205:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776259.420207:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776259.420210:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420212:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420213:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420216:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776259.420218:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776259.420220:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420222:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776259.420224:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776259.420230:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776259.420232:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420234:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776259.420238:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776259.420242:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420244:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776259.420247:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776259.420249:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776259.420251:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776259.420253:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420254:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776259.420256:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776259.420257:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776259.420260:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776259.420264:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776259.420268:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099728640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776259.420275:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776259.420277:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776259.420280:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099728640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776259.420292:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776259.420294:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776259.420298:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776259.420300:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776259.420301:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776259.420304:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776259.420306:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17974 00000400:00000200:0.0:1775776259.420310:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776259.420312:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776259.420314:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776259.420317:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420319:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420321:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420323:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776259.420324:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776259.420325:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776259.420326:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776259.420328:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776259.420329:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776259.420332:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776259.420334:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420336:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420338:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420340:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776259.420342:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776259.420344:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420345:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776259.420347:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776259.420349:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776259.420351:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420353:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776259.420355:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776259.420357:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420358:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776259.420362:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776259.420364:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776259.420366:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776259.420367:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420368:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776259.420370:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776259.420371:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776259.420374:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776259.420377:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776259.420380:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099728768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776259.420387:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776259.420390:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776259.420393:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099728768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776259.420400:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776259.420402:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776259.420405:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776259.420407:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776259.420408:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1775776259.420409:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776259.420411:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:2.1:1775776259.420412:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00080000:0.0:1775776259.420413:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17974 00000400:00000200:0.0:1775776259.420417:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776259.420419:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776259.420422:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776259.420425:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420427:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420428:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420430:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776259.420432:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776259.420433:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776259.420434:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776259.420436:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776259.420445:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776259.420447:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776259.420450:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420452:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420454:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420456:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776259.420458:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776259.420460:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420463:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776259.420465:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776259.420467:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776259.420469:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420471:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776259.420473:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776259.420475:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420477:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776259.420480:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776259.420482:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776259.420484:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776259.420486:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420487:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776259.420489:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776259.420490:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776259.420493:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776259.420496:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776259.420499:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099728896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776259.420505:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776259.420507:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776259.420510:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099728896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776259.420517:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776259.420519:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776259.420522:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776259.420524:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776259.420525:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776259.420528:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776259.420530:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17974 00000400:00000200:0.0:1775776259.420534:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776259.420536:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776259.420538:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776259.420541:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420543:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420545:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420547:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776259.420549:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776259.420550:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776259.420552:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776259.420554:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776259.420556:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776259.420558:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776259.420561:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420563:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776259.420565:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420567:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776259.420569:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776259.420572:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776259.420574:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776259.420575:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776259.420577:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776259.420579:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420581:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776259.420587:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776259.420589:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420590:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776259.420594:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776259.420596:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776259.420598:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776259.420599:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776259.420601:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776259.420603:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776259.420604:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776259.420609:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776259.420613:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776259.420616:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099729024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776259.420623:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776259.420625:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776259.420628:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099729024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776259.420636:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776259.420640:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17984) 00000800:00000200:2.1:1775776259.420844:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776259.420846:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776259.421068:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776259.421069:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776259.421273:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776259.421275:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776259.421431:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776259.421432:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775776260.188049:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776260.188053:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776260.188057:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776260.699904:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312648437, at 4312648448 00000100:00000010:0.0:1775776260.699925:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000040:1.0:1775776260.699928:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312648437, at 4312648448 00000100:00000010:1.0:1775776260.699937:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775776261.212036:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776261.212039:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776261.212042:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776262.235994:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776262.235998:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776262.236001:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776263.260033:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776263.260037:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776263.260039:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776264.284036:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776264.284040:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776264.284043:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776264.540548:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776264.540555:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776264.541472:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776264.541478:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776264.541987:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776264.541988:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776264.542196:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776264.542211:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776264.542396:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776264.542397:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775776265.309115:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776265.309117:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776265.309120:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776265.819915:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312653557, at 4312653568 00000100:00000040:0.0:1775776265.819924:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312653557, at 4312653568 00000100:00000010:1.0:1775776265.819931:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000010:0.0:1775776265.819933:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776266.332102:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776266.332106:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776266.332110:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776267.355955:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776267.355959:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776267.355963:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776268.379970:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776268.379974:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776268.379979:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776269.404032:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776269.404036:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776269.404038:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776269.659910:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776269.659918:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776269.659921:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776269.659924:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776269.659928:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776269.659932:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17984 00000400:00000200:0.0:1775776269.659941:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776269.659945:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776269.659948:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776269.659950:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.659952:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.659954:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.659955:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776269.659958:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776269.659959:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776269.659961:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776269.659963:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776269.659964:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776269.659966:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776269.659967:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.659969:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.659970:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.659971:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776269.659973:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776269.659975:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776269.659976:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776269.659979:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776269.659980:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776269.659982:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.659984:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776269.659986:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776269.659993:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776269.659996:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776269.659999:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776269.660000:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660002:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776269.660004:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776269.660007:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776269.660012:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776269.660017:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776269.660024:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099729792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776269.660031:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776269.660034:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776269.660037:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099729792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776269.660052:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776269.660054:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776269.660056:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776269.660057:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776269.660058:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776269.660060:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776269.660061:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17984 00000400:00000200:0.0:1775776269.660063:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776269.660065:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776269.660066:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776269.660068:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660069:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.660070:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660072:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776269.660074:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776269.660075:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776269.660076:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776269.660077:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776269.660078:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776269.660080:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776269.660082:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660083:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.660084:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660085:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776269.660086:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776269.660088:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660089:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776269.660090:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776269.660094:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776269.660095:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660096:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776269.660100:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776269.660102:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660103:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776269.660105:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776269.660107:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776269.660108:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776269.660109:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660110:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776269.660111:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776269.660112:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776269.660114:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776269.660116:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776269.660119:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099729920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776269.660128:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776269.660133:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776269.660138:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099729920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776269.660148:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776269.660155:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776269.660171:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776269.660176:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776269.660178:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776269.660181:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776269.660189:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17984 00000400:00000200:0.0:1775776269.660194:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776269.660204:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776269.660213:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776269.660220:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660229:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.660235:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660238:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776269.660246:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776269.660248:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776269.660254:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776269.660258:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776269.660265:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776269.660267:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776269.660272:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660274:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.660275:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660278:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776269.660283:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776269.660290:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660296:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776269.660305:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776269.660312:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776269.660321:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660328:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776269.660331:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776269.660337:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660342:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776269.660347:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776269.660355:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776269.660360:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776269.660365:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660371:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776269.660379:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776269.660385:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776269.660391:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776269.660397:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776269.660410:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099730048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776269.660441:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776269.660442:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776269.660448:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099730048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776269.660481:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776269.660492:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776269.660496:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776269.660499:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776269.660500:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776269.660502:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776269.660510:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17984 00000400:00000200:0.0:1775776269.660523:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776269.660532:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000800:00000200:2.1:1775776269.660535:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:0.0:1775776269.660538:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1775776269.660543:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000040:0.0:1775776269.660551:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660558:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.660564:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660571:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776269.660575:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776269.660580:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776269.660586:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776269.660589:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776269.660594:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776269.660606:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776269.660615:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660626:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.660635:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660645:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776269.660653:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776269.660664:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660673:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776269.660682:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776269.660688:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776269.660694:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660696:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776269.660700:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776269.660707:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660715:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776269.660726:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776269.660733:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776269.660736:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776269.660740:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.660742:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776269.660743:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776269.660745:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776269.660748:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776269.660752:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776269.660757:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099730176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776269.660777:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776269.660817:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776269.660824:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099730176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776269.660843:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776269.660849:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776269.660856:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776269.660860:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776269.660868:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776269.660872:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776269.660884:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17984 00000400:00000200:0.0:1775776269.660896:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776269.660908:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776269.660918:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776269.660928:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660939:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.660947:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660949:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776269.660950:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776269.660951:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776269.660952:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776269.660954:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776269.660960:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776269.660964:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776269.660972:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660978:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776269.660986:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776269.660994:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776269.660999:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776269.661010:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776269.661015:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776269.661024:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776269.661032:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776269.661043:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.661054:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776269.661063:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776269.661067:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.661070:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776269.661076:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776269.661081:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776269.661088:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776269.661090:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776269.661100:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776269.661104:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776269.661110:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776269.661120:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776269.661125:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776269.661139:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099730304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776269.661152:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776269.661160:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776269.661172:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099730304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776269.661195:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776269.661199:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (17995) 00000800:00000200:2.1:1775776269.661321:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776269.661322:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776269.661490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776269.661491:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776269.661679:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776269.661680:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776269.661887:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776269.661888:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775776270.428080:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776270.428084:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776270.428088:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776270.939880:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312658677, at 4312658688 00000100:00000040:1.0:1775776270.939887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312658677, at 4312658688 00000100:00000010:1.0:1775776270.939902:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000010:0.0:1775776270.939905:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775776271.452015:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776271.452018:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776271.452022:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776272.476012:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776272.476016:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776272.476019:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776273.499987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776273.499991:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776273.499994:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776274.523923:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776274.523927:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776274.523929:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776274.779904:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776274.779915:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776274.779918:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776274.779922:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776274.779926:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776274.779933:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 17990 00000400:00000200:0.0:1775776274.779942:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776274.779948:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776274.779950:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776274.779952:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.779954:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.779956:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.779957:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776274.779960:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776274.779961:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776274.779963:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776274.779965:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776274.779967:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776274.779968:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776274.779970:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.779971:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.779972:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.779974:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776274.779977:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776274.779979:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776274.779982:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776274.779984:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776274.779987:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776274.779990:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.779992:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776274.779994:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776274.780003:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775776274.780006:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776274.780011:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776274.780013:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780016:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776274.780018:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776274.780022:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776274.780029:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776274.780035:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776274.780042:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099730432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776274.780049:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776274.780052:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776274.780055:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099730432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776274.780073:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776274.780076:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776274.780078:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776274.780079:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776274.780080:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776274.780082:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776274.780083:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17990 00000400:00000200:0.0:1775776274.780086:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776274.780088:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776274.780090:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776274.780092:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780093:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.780095:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780097:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776274.780098:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776274.780100:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776274.780101:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776274.780102:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776274.780104:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776274.780105:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776274.780109:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780111:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.780114:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780116:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776274.780119:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776274.780122:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780126:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776274.780130:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776274.780135:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776274.780139:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780141:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776274.780145:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776274.780149:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780150:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776274.780153:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776274.780154:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776274.780156:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776274.780157:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780160:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776274.780163:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776274.780165:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776274.780168:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776274.780172:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776274.780178:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099730560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776274.780196:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776274.780199:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776274.780202:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099730560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776274.780208:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776274.780213:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776274.780219:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776274.780221:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776274.780223:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776274.780226:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776274.780231:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17990 00000400:00000200:0.0:1775776274.780238:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776274.780242:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776274.780246:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776274.780251:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780253:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.780257:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780258:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776274.780260:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776274.780262:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776274.780264:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776274.780266:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776274.780267:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776274.780269:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776274.780273:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780275:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.780278:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780282:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776274.780284:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776274.780287:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780291:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776274.780293:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776274.780295:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776274.780298:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780300:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776274.780302:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776274.780306:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780310:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776274.780314:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776274.780316:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776274.780317:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776274.780319:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780323:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776274.780326:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776274.780329:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776274.780334:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776274.780338:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776274.780346:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099730688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776274.780360:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1775776274.780364:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776274.780365:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776274.780368:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000040:0.0:1775776274.780373:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099730688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776274.780386:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776274.780389:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776274.780395:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776274.780398:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776274.780401:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776274.780406:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776274.780412:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17990 00000400:00000200:0.0:1775776274.780428:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776274.780436:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776274.780439:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776274.780446:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780450:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.780455:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780458:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776274.780461:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776274.780463:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776274.780467:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776274.780469:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776274.780471:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776274.780474:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776274.780479:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780482:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.780485:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780489:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776274.780491:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776274.780494:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780497:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776274.780501:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776274.780505:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776274.780508:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780512:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776274.780516:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776274.780519:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780521:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776274.780524:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776274.780529:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776274.780535:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776274.780537:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780541:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776274.780544:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776274.780547:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776274.780553:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776274.780559:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776274.780567:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099730816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776274.780584:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776274.780589:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776274.780596:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099730816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776274.780612:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776274.780618:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776274.780625:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776274.780630:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776274.780633:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776274.780639:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776274.780645:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 17990 00000400:00000200:0.0:1775776274.780654:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776274.780662:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776274.780665:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776274.780673:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780678:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.780682:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780684:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776274.780687:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776274.780690:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776274.780691:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776274.780695:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776274.780698:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776274.780703:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776274.780708:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780712:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776274.780716:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780720:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776274.780726:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776274.780731:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776274.780737:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776274.780753:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776274.780757:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776274.780762:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780767:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776274.780771:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776274.780775:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780798:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776274.780803:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776274.780807:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776274.780813:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776274.780816:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776274.780820:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776274.780824:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776274.780827:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776274.780834:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776274.780840:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776274.780848:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099730944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776274.780865:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776274.780869:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776274.780878:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099730944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776274.780897:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776274.780905:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18000) 00000800:00000200:2.1:1775776274.780937:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776274.780938:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776274.781112:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776274.781114:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776274.781265:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776274.781266:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776274.781477:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776274.781480:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775776275.547942:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776275.547945:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776275.547948:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776276.059902:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312663797, at 4312663808 00000100:00000040:1.0:1775776276.059912:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312663797, at 4312663808 00000100:00000010:0.0:1775776276.059920:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000010:1.0:1775776276.059921:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775776276.572005:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776276.572009:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776276.572012:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776277.595950:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776277.595953:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776277.595955:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776278.619963:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776278.619966:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776278.619973:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776279.643977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776279.643981:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776279.643984:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776279.900519:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776279.900535:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776279.901144:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776279.901145:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776279.901310:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776279.901311:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776279.901487:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776279.901489:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776279.901834:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776279.901836:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775776280.668002:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776280.668006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776280.668009:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776281.179924:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312668917, at 4312668928 00000100:00000010:1.0:1775776281.179941:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000040:0.0:1775776281.180015:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312668917, at 4312668928 00000100:00000010:0.0:1775776281.180032:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775776281.691932:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776281.691935:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776281.691937:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776282.715946:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776282.715949:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776282.715953:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776283.739924:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776283.739926:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776283.739928:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776284.763967:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776284.763970:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776284.763974:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776285.020346:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776285.020351:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776285.020916:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776285.020917:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776285.021095:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776285.021096:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776285.021272:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776285.021275:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776285.021515:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776285.021517:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775776285.787989:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776285.787993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776285.787997:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776286.299878:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312674037, at 4312674048 00000100:00000040:1.0:1775776286.299890:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312674037, at 4312674048 00000100:00000010:0.0:1775776286.299892:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000010:1.0:1775776286.299898:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775776286.811966:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776286.811969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776286.811971:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776287.835975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776287.835979:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776287.835982:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776288.859995:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776288.859999:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776288.860002:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776289.883982:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776289.883986:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776289.883990:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776290.140412:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776290.140416:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776290.140978:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776290.140980:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776290.141151:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776290.141166:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776290.141346:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776290.141348:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776290.141590:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776290.141592:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1775776290.846590:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776290.846610:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776290.846612:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776290.846617:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776290.846621:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776290.846624:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776290.846627:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776290.846629:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776290.846631:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776290.907925:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776290.907928:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776290.907931:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776291.419881:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312679157, at 4312679168 00000100:00000040:1.0:1775776291.419892:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312679157, at 4312679168 00000100:00000010:0.0:1775776291.419894:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000010:1.0:1775776291.419899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776291.931986:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776291.931989:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776291.931992:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776292.892771:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776292.892810:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776292.892813:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776292.892827:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776292.892831:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776292.892836:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776292.892838:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776292.892841:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776292.892844:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776292.956023:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776292.956027:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776292.956031:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776293.979970:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776293.979974:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776293.979977:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776295.004033:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776295.004037:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776295.004040:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776295.260425:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776295.260431:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776295.260969:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776295.260972:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776295.261255:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776295.261257:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776295.261549:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776295.261551:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776295.261838:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776295.261840:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:0.0:1775776295.965483:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776295.965489:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776295.965491:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776295.965496:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776295.965499:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776295.965502:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776295.965505:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776295.965508:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776295.965511:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776296.027988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776296.027992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776296.027996:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776296.539967:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312684277, at 4312684288 00000100:00000040:1.0:1775776296.539967:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312684277, at 4312684288 00000100:00000010:1.0:1775776296.539992:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000100:00000010:0.0:1775776296.539998:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503449c0. 00000100:00000001:0.0:1775776296.987917:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776296.987928:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776296.987930:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776296.987937:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776296.987942:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776296.987946:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776296.987949:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776296.987952:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776296.987956:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776297.051987:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776297.051990:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776297.051993:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776298.011883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776298.011889:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776298.011891:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776298.011896:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776298.011899:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776298.011902:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776298.011904:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776298.011906:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776298.011908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776298.075982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776298.075985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776298.075988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776299.035899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776299.035906:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776299.035908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776299.035913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776299.035917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776299.035921:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776299.035924:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776299.035927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776299.035929:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776299.100012:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776299.100015:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776299.100017:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776300.059858:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776300.059866:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776300.059868:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776300.059879:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776300.059883:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776300.059886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776300.059889:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776300.059892:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776300.059894:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776300.123986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776300.123989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776300.123993:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776300.380379:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776300.380386:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776300.381043:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776300.381045:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776300.381266:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776300.381268:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776300.381596:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776300.381599:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776300.381875:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776300.381878:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1775776301.083909:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776301.083916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776301.083918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776301.083924:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776301.083928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776301.083932:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776301.083935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776301.083937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776301.083939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776301.147937:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776301.147941:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776301.147943:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776301.659958:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312689397, at 4312689408 00000100:00000010:0.0:1775776301.659972:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000040:1.0:1775776301.659996:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312689397, at 4312689408 00000100:00000010:1.0:1775776301.660002:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:0.0:1775776302.107885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776302.107890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776302.107892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776302.107896:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776302.107898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776302.107901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776302.107903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776302.107904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776302.107906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776302.171971:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776302.171974:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776302.171976:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776303.131897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776303.131903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776303.131904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776303.131908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776303.131912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776303.131916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776303.131918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776303.131921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776303.131923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776303.195999:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776303.196004:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776303.196008:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776304.155869:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776304.155876:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776304.155878:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776304.155884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776304.155888:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776304.155892:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776304.155895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776304.155897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776304.155900:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776304.219946:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776304.219949:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776304.219951:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776305.179907:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776305.179915:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776305.179916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776305.179922:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776305.179925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776305.179929:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776305.179932:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776305.179934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776305.179937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776305.244075:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776305.244079:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776305.244082:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776305.501232:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776305.501237:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776305.501891:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776305.501894:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776305.506536:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776305.506539:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776305.506861:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776305.506863:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776305.507104:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776305.507106:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1775776306.203890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776306.203898:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776306.203900:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776306.203905:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776306.203908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776306.203912:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776306.203914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776306.203917:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776306.203919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776306.268025:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776306.268029:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776306.268032:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776306.779966:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312694518, at 4312694528 00000100:00000010:0.0:1775776306.779987:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000100:00000040:1.0:1775776306.780253:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312694518, at 4312694528 00000100:00000010:1.0:1775776306.780261:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503440c0. 00000100:00000001:0.0:1775776307.227917:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776307.227925:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776307.227926:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776307.227931:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776307.227934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776307.227938:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776307.227940:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776307.227942:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776307.227944:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776307.292039:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776307.292045:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776307.292050:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776308.251838:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776308.251844:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776308.251846:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776308.251850:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776308.251852:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776308.251856:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776308.251858:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776308.251860:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776308.251862:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776308.315952:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776308.315956:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776308.315959:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776309.275909:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776309.275915:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776309.275916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776309.275921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776309.275924:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776309.275928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776309.275930:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776309.275933:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776309.275936:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776309.339992:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776309.339995:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776309.339997:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776310.299948:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776310.299955:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776310.299957:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776310.299963:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776310.299967:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776310.299970:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776310.299973:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776310.299975:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776310.299978:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776310.363929:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776310.363932:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776310.363934:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776310.620579:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776310.621028:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776310.621191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776310.621341:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776310.621475:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776311.323961:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776311.323967:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776311.323970:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776311.323975:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776311.323978:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776311.323982:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776311.323985:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776311.323987:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776311.323989:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776311.388138:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776311.388141:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776311.388145:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776311.899867:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312699637, at 4312699648 00000100:00000040:0.0:1775776311.899878:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312699637, at 4312699648 00000100:00000010:1.0:1775776311.899885:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000100:00000010:0.0:1775776311.899887:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000100:00000001:0.0:1775776312.348233:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776312.348242:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776312.348244:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776312.348249:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776312.348253:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776312.348258:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776312.348260:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776312.348262:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776312.348265:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776312.412010:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776312.412012:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776312.412014:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776313.371955:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776313.371967:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776313.371969:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776313.371978:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776313.371983:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776313.371989:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776313.371992:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776313.371995:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776313.371999:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776313.436030:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776313.436036:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776313.436041:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775776313.692033:0:301876:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775776313.692088:0:301876:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775776313.692099:0:301876:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776313.692103:0:301876:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775776313.692107:0:301876:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776313.692111:0:301876:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775776313.692116:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776313.692120:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511b00. 00000400:00000200:1.0:1775776313.692125:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776313.692130:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776313.692136:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1f68 00000400:00000010:1.0:1775776313.692139:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1f68. 00000100:00000001:1.0:1775776313.692143:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776313.692153:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7ea00 x1862033099734912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692179:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776313.692184:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ea00 x1862033099734912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692191:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.692193:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776313.692203:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776313.692206:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511700. 00000400:00000200:1.0:1775776313.692209:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776313.692212:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776313.692214:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:1.0:1775776313.692222:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:1.0:1775776313.692225:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776313.692229:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7fb80 x1862033099735040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692237:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776313.692239:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7fb80 x1862033099735040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692245:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.692247:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776313.692251:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776313.692253:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511f00. 00000400:00000200:1.0:1775776313.692255:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776313.692258:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776313.692270:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1908 00000400:00000010:1.0:1775776313.692272:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1908. 00000100:00000001:1.0:1775776313.692275:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776313.692279:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7e680 x1862033099735168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692287:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776313.692290:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e680 x1862033099735168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692296:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.692297:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776313.692302:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776313.692304:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511400. 00000400:00000200:1.0:1775776313.692306:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776313.692309:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776313.692312:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:1.0:1775776313.692314:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:1.0:1775776313.692316:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776313.692320:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7c000 x1862033099735296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692327:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776313.692329:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c000 x1862033099735296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692335:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.692336:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776313.692342:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776313.692344:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511500. 00000400:00000200:1.0:1775776313.692346:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776313.692354:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776313.692357:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1330 00000400:00000010:1.0:1775776313.692359:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1330. 00000100:00000001:1.0:1775776313.692362:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776313.692365:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7e300 x1862033099735424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692373:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776313.692375:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e300 x1862033099735424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776335 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.692381:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.692383:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776314.395903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776314.395910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776314.395912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776314.395917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776314.395920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776314.395924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776314.395927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776314.395929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776314.395931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776314.460212:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776314.460217:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776314.460220:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776315.419996:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776315.420168:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776315.420170:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776315.420176:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776315.420180:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776315.420184:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776315.420186:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776315.420188:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776315.420191:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776315.484080:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776315.484083:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776315.484086:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776315.739901:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776315.739909:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776315.739911:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776315.739914:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776315.739918:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776315.739923:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18031 00000400:00000200:0.0:1775776315.739931:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776315.739935:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776315.739938:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776315.739940:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.739942:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.739944:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.739945:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776315.739948:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776315.739949:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776315.739951:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776315.739953:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776315.739955:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776315.739957:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776315.739960:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.739962:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.739964:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.739966:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776315.739968:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776315.739971:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776315.739974:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776315.739976:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776315.739979:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776315.739982:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.739985:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776315.739987:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776315.739995:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776315.739999:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776315.740003:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776315.740004:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740007:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776315.740010:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776315.740014:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776315.740021:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776315.740043:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776315.740052:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099735552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776315.740062:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776315.740067:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776315.740072:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099735552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776315.740104:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776315.740107:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776315.740111:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776315.740112:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776315.740114:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776315.740117:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776315.740119:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18031 00000400:00000200:0.0:1775776315.740123:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776315.740127:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776315.740129:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776315.740132:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740134:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.740135:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740137:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776315.740139:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776315.740141:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776315.740142:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776315.740144:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776315.740145:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776315.740147:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776315.740150:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740152:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.740154:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740156:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776315.740159:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776315.740161:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740163:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776315.740165:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776315.740170:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776315.740172:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740174:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776315.740178:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776315.740180:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740182:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776315.740185:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776315.740188:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776315.740190:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776315.740191:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740192:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776315.740194:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776315.740195:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776315.740199:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776315.740202:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776315.740206:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099735680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776315.740213:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776315.740215:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776315.740218:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099735680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776315.740225:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776315.740227:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776315.740230:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776315.740232:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776315.740233:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776315.740235:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776315.740238:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18031 00000400:00000200:0.0:1775776315.740241:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776315.740244:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776315.740246:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776315.740249:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740251:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.740252:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740254:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776315.740256:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776315.740257:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776315.740259:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776315.740261:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776315.740262:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776315.740264:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776315.740267:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740269:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.740271:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740272:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776315.740274:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776315.740277:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740279:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776315.740280:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776315.740283:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776315.740285:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740287:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776315.740288:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776315.740290:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740292:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776315.740295:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776315.740297:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776315.740299:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776315.740300:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740302:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776315.740303:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776315.740305:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776315.740308:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776315.740311:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776315.740316:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099735808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776315.740322:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776315.740324:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776315.740328:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099735808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776315.740334:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776315.740336:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776315.740339:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776315.740341:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776315.740342:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776315.740345:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776315.740347:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18031 00000400:00000200:0.0:1775776315.740351:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776315.740353:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776315.740356:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776315.740358:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740360:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.740362:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740364:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776315.740365:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776315.740367:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776315.740369:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776315.740370:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776315.740373:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776315.740375:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776315.740377:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740379:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.740380:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740382:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776315.740385:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776315.740387:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740390:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776315.740391:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776315.740394:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776315.740396:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740398:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776315.740400:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776315.740402:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740404:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776315.740407:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776315.740409:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776315.740411:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776315.740412:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740414:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776315.740415:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776315.740417:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776315.740420:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776315.740423:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776315.740427:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099735936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776315.740433:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776315.740435:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776315.740438:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099735936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776315.740444:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776315.740446:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776315.740449:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776315.740450:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776315.740451:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776315.740454:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776315.740456:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18031 00000400:00000200:0.0:1775776315.740459:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776315.740462:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776315.740464:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776315.740466:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740469:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.740470:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740472:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776315.740474:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776315.740475:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776315.740477:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776315.740478:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776315.740480:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776315.740482:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776315.740484:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740486:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776315.740488:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740489:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776315.740491:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776315.740493:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776315.740495:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776315.740497:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776315.740500:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776315.740501:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740503:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776315.740505:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776315.740507:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740508:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776315.740511:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776315.740513:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776315.740515:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776315.740516:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776315.740518:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776315.740519:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776315.740520:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776315.740523:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776315.740527:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776315.740530:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099736064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776315.740536:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776315.740538:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776315.740541:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099736064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776315.740548:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776315.740552:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18041) 00000800:00000200:3.1:1775776315.740896:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776315.740899:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776315.741232:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776315.741234:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776315.741488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776315.741490:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776315.741740:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776315.741742:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776315.742016:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776315.742019:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1775776316.443889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776316.443897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776316.443899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776316.443904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776316.443907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776316.443911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776316.443914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776316.443917:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776316.443919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776316.507955:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776316.507957:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776316.507960:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776317.019867:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312704758, at 4312704768 00000100:00000010:1.0:1775776317.019885:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000100:00000040:0.0:1775776317.019946:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312704758, at 4312704768 00000100:00000010:0.0:1775776317.019953:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503447e0. 00000100:00000001:0.0:1775776317.467910:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776317.467916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776317.467917:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776317.467921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776317.467924:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776317.467928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776317.467930:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776317.467932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776317.467934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776317.532019:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776317.532023:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776317.532026:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776318.491913:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776318.491919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776318.491921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776318.491927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776318.491931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776318.491936:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776318.491939:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776318.491941:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776318.491945:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776318.556054:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776318.556057:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776318.556059:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776319.515906:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776319.515912:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776319.515914:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776319.515919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776319.515922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776319.515926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776319.515928:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776319.515930:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776319.515933:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776319.579982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776319.579985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776319.579987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776320.539844:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776320.539849:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776320.539850:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776320.539854:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776320.539856:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776320.539859:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776320.539861:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776320.539863:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776320.539865:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776320.603921:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776320.603924:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776320.603927:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776320.860414:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776320.860419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776320.861067:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776320.861069:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776320.861344:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776320.861346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776320.861601:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776320.861603:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776320.861870:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776320.861883:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1775776321.563910:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776321.563917:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776321.563919:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776321.563939:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776321.563943:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776321.563948:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776321.563950:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776321.563952:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776321.563955:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776321.627982:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776321.627985:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776321.627987:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776322.139876:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312709877, at 4312709888 00000100:00000040:0.0:1775776322.139888:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312709877, at 4312709888 00000100:00000010:1.0:1775776322.139888:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000010:0.0:1775776322.139895:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:0.0:1775776322.587881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776322.587887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776322.587889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776322.587894:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776322.587897:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776322.587901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776322.587912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776322.587915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776322.587917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776322.651958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776322.651961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776322.651965:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776323.611871:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776323.611878:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776323.611880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776323.611885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776323.611903:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776323.611911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776323.611913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776323.611915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776323.611917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776323.676045:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776323.676047:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776323.676050:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776324.635899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776324.635906:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776324.635907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776324.635912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776324.635915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776324.635919:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776324.635922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776324.635924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776324.635926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776324.700026:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776324.700029:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776324.700033:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776325.659904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776325.659910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776325.659912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776325.659915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776325.659918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776325.659921:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776325.659922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776325.659924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776325.659926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776325.723979:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776325.723984:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776325.723987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776325.979918:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776325.979926:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776325.979928:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776325.979932:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776325.979935:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776325.979942:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18041 00000400:00000200:0.0:1775776325.979950:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776325.979955:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776325.979957:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776325.979959:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.979961:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.979963:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.979965:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776325.979967:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776325.979969:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776325.979971:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776325.979972:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776325.979974:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776325.979975:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776325.979977:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.979978:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.979979:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.979981:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776325.979983:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776325.979985:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776325.979986:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776325.979988:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776325.979991:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776325.979993:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.979995:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776325.979996:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776325.980005:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776325.980008:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776325.980012:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776325.980013:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980016:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776325.980018:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776325.980022:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776325.980030:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776325.980037:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776325.980046:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099736832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776325.980057:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776325.980060:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776325.980063:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099736832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776325.980080:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776325.980082:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776325.980084:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776325.980085:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776325.980086:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776325.980088:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776325.980090:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18041 00000400:00000200:0.0:1775776325.980093:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776325.980094:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776325.980096:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776325.980099:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980100:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.980102:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980104:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776325.980105:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776325.980106:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776325.980107:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776325.980108:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776325.980110:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776325.980113:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776325.980116:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980118:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.980120:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980123:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776325.980125:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776325.980130:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980135:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776325.980139:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776325.980143:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776325.980146:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980148:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776325.980153:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776325.980156:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980160:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776325.980163:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776325.980166:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776325.980170:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776325.980173:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980175:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776325.980176:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776325.980177:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776325.980181:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776325.980184:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776325.980190:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099736960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776325.980196:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776325.980198:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776325.980202:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099736960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776325.980208:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776325.980209:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776325.980212:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776325.980213:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776325.980215:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776325.980217:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776325.980220:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18041 00000400:00000200:0.0:1775776325.980225:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776325.980227:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776325.980229:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776325.980232:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980236:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.980239:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980243:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776325.980246:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776325.980248:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776325.980250:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776325.980252:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776325.980254:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776325.980258:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776325.980264:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980269:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.980271:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980276:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776325.980278:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776325.980283:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980288:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776325.980290:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776325.980293:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776325.980297:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980300:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776325.980302:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776325.980308:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980310:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776325.980313:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776325.980316:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776325.980319:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776325.980321:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980322:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776325.980324:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776325.980333:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776325.980339:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776325.980343:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776325.980347:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099737088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776325.980363:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776325.980366:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776325.980370:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099737088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776325.980378:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776325.980382:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776325.980387:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776325.980389:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776325.980392:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1775776325.980395:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776325.980396:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776325.980400:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18041 00000800:00000200:2.1:1775776325.980400:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000400:00000200:0.0:1775776325.980407:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776325.980411:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776325.980414:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776325.980416:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980418:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.980420:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980428:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776325.980432:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776325.980434:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776325.980436:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776325.980437:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776325.980439:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776325.980442:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776325.980448:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980452:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.980455:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980457:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776325.980459:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776325.980463:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980467:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776325.980471:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776325.980474:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776325.980477:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980479:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776325.980482:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776325.980484:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980486:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776325.980490:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776325.980493:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776325.980495:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776325.980497:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980499:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776325.980499:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776325.980501:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776325.980505:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776325.980507:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776325.980511:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099737216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776325.980517:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776325.980521:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776325.980526:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099737216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776325.980535:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776325.980537:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776325.980546:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776325.980547:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776325.980549:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776325.980553:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776325.980557:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18041 00000400:00000200:0.0:1775776325.980563:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776325.980567:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776325.980569:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776325.980573:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980575:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.980578:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980583:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776325.980585:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776325.980586:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776325.980588:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776325.980590:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776325.980594:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776325.980595:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776325.980607:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980612:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776325.980617:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980620:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776325.980624:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776325.980631:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776325.980636:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776325.980640:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776325.980645:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776325.980649:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980652:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776325.980654:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776325.980658:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980661:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776325.980665:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776325.980667:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776325.980670:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776325.980672:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776325.980676:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776325.980678:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776325.980688:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776325.980693:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776325.980699:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776325.980708:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099737344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776325.980736:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776325.980740:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776325.980745:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099737344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776325.980757:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776325.980762:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18051) 00000800:00000200:2.1:1775776325.980926:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776325.980927:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776325.981089:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776325.981090:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776325.981239:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776325.981241:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776325.981433:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776325.981434:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1775776326.683891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776326.683896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776326.683897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776326.683901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776326.683904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776326.683907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776326.683909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776326.683911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776326.683913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776326.748020:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776326.748022:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776326.748024:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776327.259876:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312714997, at 4312715008 00000100:00000010:1.0:1775776327.259889:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000040:0.0:1775776327.259899:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312714997, at 4312715008 00000100:00000010:0.0:1775776327.259905:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000001:0.0:1775776327.707853:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776327.707859:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776327.707860:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776327.707864:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776327.707866:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776327.707869:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776327.707871:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776327.707872:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776327.707874:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776327.771961:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776327.771964:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776327.771968:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776328.731904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776328.731911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776328.731913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776328.731920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776328.731923:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776328.731928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776328.731931:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776328.731934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776328.731938:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776328.796058:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776328.796062:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776328.796065:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776329.755881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776329.755890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776329.755892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776329.755897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776329.755901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776329.755905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776329.755908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776329.755910:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776329.755913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776329.819956:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776329.819959:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776329.819962:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776330.779877:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776330.779883:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776330.779885:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776330.779889:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776330.779892:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776330.779896:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776330.779898:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776330.779900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776330.779902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776330.843975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776330.843980:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776330.843983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776331.100508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776331.100515:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775776331.101090:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776331.101093:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775776331.101338:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776331.101341:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775776331.101555:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776331.101557:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775776331.101805:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776331.101807:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000001:0.0:1775776331.803894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776331.803900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776331.803902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776331.803907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776331.803910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776331.803913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776331.803916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776331.803919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776331.803921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776331.867964:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776331.867967:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776331.867970:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776332.379890:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312720117, at 4312720128 00000100:00000010:1.0:1775776332.379903:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503445a0. 00000100:00000040:0.0:1775776332.379936:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312720117, at 4312720128 00000100:00000010:0.0:1775776332.379945:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503442a0. 00000100:00000001:0.0:1775776332.827913:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776332.827922:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776332.827924:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776332.827930:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776332.827934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776332.827938:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776332.827941:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776332.827944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776332.827947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776332.891999:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776332.892003:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776332.892006:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776333.851896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776333.851902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776333.851904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776333.851908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776333.851911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776333.851914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776333.851917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776333.851919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776333.851921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776333.915950:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776333.915955:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776333.915959:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776334.875887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776334.875893:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776334.875894:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776334.875898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776334.875901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776334.875905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776334.875907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776334.875909:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776334.875911:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776334.939961:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776334.939964:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776334.939966:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776335.899932:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776335.899942:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776335.899944:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776335.899952:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776335.899957:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776335.899963:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776335.899966:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776335.899969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776335.899972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776335.963979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776335.963983:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776335.963987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776336.220543:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776336.220550:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776336.221154:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776336.221157:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776336.221461:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776336.221464:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776336.221765:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776336.221768:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776336.222065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776336.222067:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000100:00000001:0.0:1775776336.923861:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776336.923867:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776336.923869:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776336.923873:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776336.923876:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776336.923879:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776336.923881:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776336.923883:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776336.923885:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776336.987973:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776336.987976:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776336.987978:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776337.499892:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312725237, at 4312725248 00000100:00000010:0.0:1775776337.499909:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000100:00000040:1.0:1775776337.499917:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312725237, at 4312725248 00000100:00000010:1.0:1775776337.499926:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c60. 00000100:00000001:0.0:1775776337.947908:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776337.947914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776337.947915:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776337.947920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776337.947923:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776337.947926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776337.947928:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776337.947930:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776337.947932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776338.011957:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776338.011961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776338.011964:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776338.971856:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776338.971862:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776338.971864:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776338.971868:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776338.971871:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776338.971873:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776338.971876:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776338.971878:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776338.971879:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776339.036051:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776339.036055:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776339.036058:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776339.995962:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776339.995970:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776339.995972:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776339.995978:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776339.995982:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776339.995986:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776339.995989:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776339.995991:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776339.995995:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776340.059933:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776340.059937:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776340.059939:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776341.019903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776341.019909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776341.019911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776341.019915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776341.019918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776341.019921:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776341.019923:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776341.019925:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776341.019928:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776341.083987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776341.083990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776341.083993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776341.339934:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776341.339942:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776341.339945:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776341.339949:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776341.339954:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776341.339961:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18056 00000400:00000200:0.0:1775776341.339976:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776341.339983:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776341.339986:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776341.339990:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.339994:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.339997:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340000:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776341.340004:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776341.340006:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776341.340008:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776341.340010:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776341.340012:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776341.340014:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776341.340016:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340017:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340018:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340019:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776341.340021:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776341.340024:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340025:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776341.340028:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776341.340030:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776341.340033:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340035:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776341.340037:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776341.340045:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776341.340048:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776341.340050:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776341.340052:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340054:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776341.340056:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776341.340060:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776341.340066:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776341.340072:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776341.340079:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099738752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776341.340087:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776341.340090:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776341.340093:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099738752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776341.340110:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776341.340113:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776341.340115:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776341.340117:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776341.340118:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776341.340120:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776341.340122:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18056 00000400:00000200:0.0:1775776341.340125:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776341.340128:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776341.340130:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776341.340132:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340134:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340136:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340138:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776341.340141:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776341.340142:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776341.340145:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776341.340147:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776341.340150:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776341.340153:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776341.340158:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340160:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340163:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340167:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776341.340171:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776341.340176:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340180:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776341.340182:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776341.340187:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776341.340191:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340195:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776341.340199:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776341.340202:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340205:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776341.340208:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776341.340211:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776341.340213:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776341.340216:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340217:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776341.340219:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776341.340223:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776341.340229:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776341.340235:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776341.340244:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099738880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776341.340256:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776341.340261:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776341.340270:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099738880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776341.340290:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776341.340293:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776341.340297:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776341.340301:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776341.340302:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776341.340304:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776341.340316:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18056 00000400:00000200:0.0:1775776341.340321:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776341.340325:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776341.340329:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776341.340331:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340332:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340333:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340335:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776341.340336:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776341.340338:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776341.340339:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776341.340341:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776341.340342:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776341.340344:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776341.340348:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340352:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340353:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340356:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776341.340359:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776341.340364:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340368:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776341.340371:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776341.340375:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776341.340379:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340381:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776341.340384:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776341.340390:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340393:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776341.340397:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776341.340399:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776341.340404:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776341.340407:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340413:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776341.340417:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776341.340420:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776341.340427:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776341.340431:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776341.340437:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099739008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776341.340455:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:3.1:1775776341.340458:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776341.340459:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776341.340462:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000100:00000040:0.0:1775776341.340467:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099739008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776341.340480:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776341.340483:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776341.340487:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776341.340489:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776341.340492:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776341.340494:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776341.340499:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18056 00000400:00000200:0.0:1775776341.340507:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776341.340509:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776341.340511:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776341.340513:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340514:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340516:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340517:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776341.340519:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776341.340521:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776341.340523:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776341.340526:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776341.340528:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776341.340530:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776341.340533:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340537:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340541:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340545:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776341.340550:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776341.340555:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340557:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776341.340559:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776341.340562:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776341.340565:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340568:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776341.340571:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776341.340574:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340591:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776341.340594:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776341.340596:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776341.340597:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776341.340598:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340601:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776341.340604:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776341.340606:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776341.340609:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776341.340615:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776341.340622:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099739136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776341.340629:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776341.340632:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776341.340635:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099739136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776341.340641:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776341.340646:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776341.340650:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776341.340653:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776341.340656:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776341.340660:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776341.340666:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18056 00000400:00000200:0.0:1775776341.340675:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776341.340681:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776341.340684:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776341.340690:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340696:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340698:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340702:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776341.340708:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776341.340710:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776341.340714:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776341.340717:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776341.340719:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776341.340723:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776341.340730:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340736:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776341.340740:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340743:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776341.340746:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776341.340752:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776341.340760:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776341.340765:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776341.340770:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776341.340775:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340811:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776341.340813:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776341.340818:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340821:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776341.340832:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776341.340837:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776341.340842:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776341.340844:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776341.340846:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776341.340848:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776341.340851:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776341.340855:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776341.340859:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776341.340866:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099739264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776341.340877:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776341.340881:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776341.340884:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099739264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776341.340894:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776341.340899:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18066) 00000800:00000200:3.1:1775776341.341109:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776341.341111:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776341.341310:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776341.341313:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776341.341565:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776341.341567:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776341.341809:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776341.341811:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000100:00000001:0.0:1775776342.043899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776342.043905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776342.043907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776342.043912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776342.043914:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776342.043917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776342.043919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776342.043921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776342.043923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776342.107975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776342.107979:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776342.107983:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776342.619880:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312730357, at 4312730368 00000100:00000040:0.0:1775776342.619884:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312730357, at 4312730368 00000100:00000010:1.0:1775776342.619899:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344360. 00000100:00000010:0.0:1775776342.619901:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344180. 00000100:00000001:0.0:1775776343.067872:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776343.067878:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776343.067879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776343.067884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776343.067887:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776343.067890:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776343.067892:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776343.067894:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776343.067896:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776343.131978:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776343.131981:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776343.131984:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776344.091924:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776344.091933:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776344.091935:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776344.091941:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776344.091945:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776344.091949:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776344.091951:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776344.091954:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776344.091957:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776344.156002:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776344.156006:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776344.156010:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776345.115923:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776345.115931:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776345.115934:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776345.115940:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776345.115944:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776345.115948:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776345.115951:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776345.115954:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776345.115957:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776345.179977:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776345.179981:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776345.179986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776346.139892:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776346.139899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776346.139902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776346.139908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776346.139911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776346.139915:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776346.139918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776346.139921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776346.139924:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776346.204025:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776346.204029:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776346.204033:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776346.459944:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776346.459955:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776346.459958:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776346.459963:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776346.459967:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776346.459975:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18061 00000400:00000200:0.0:1775776346.459985:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776346.459992:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776346.459995:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776346.459998:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460002:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460004:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460007:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776346.460011:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776346.460013:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776346.460016:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776346.460019:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776346.460022:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776346.460024:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776346.460028:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460030:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460032:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460035:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776346.460038:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776346.460041:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460044:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776346.460047:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776346.460050:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776346.460052:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460054:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776346.460056:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776346.460065:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776346.460068:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776346.460071:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776346.460073:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460075:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776346.460077:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776346.460081:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776346.460088:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776346.460094:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776346.460102:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099739392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776346.460110:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776346.460113:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776346.460117:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099739392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776346.460136:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776346.460138:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776346.460141:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776346.460142:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776346.460143:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776346.460153:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776346.460155:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18061 00000400:00000200:0.0:1775776346.460162:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776346.460165:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776346.460166:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776346.460170:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460172:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460175:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460178:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776346.460182:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776346.460184:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776346.460187:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776346.460190:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776346.460194:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776346.460198:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776346.460205:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460212:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460217:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460220:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776346.460224:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776346.460231:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460236:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776346.460239:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776346.460245:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776346.460249:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460251:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776346.460255:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776346.460261:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460329:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776346.460337:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776346.460341:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776346.460344:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776346.460346:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460347:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776346.460350:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776346.460353:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776346.460358:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776346.460370:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776346.460377:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099739520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776346.460390:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776346.460395:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776346.460400:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099739520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776346.460414:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776346.460418:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776346.460424:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776346.460427:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776346.460429:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776346.460434:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776346.460439:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18061 00000400:00000200:0.0:1775776346.460444:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776346.460451:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776346.460453:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776346.460463:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:3.1:1775776346.460465:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776346.460466:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460471:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:3.1:1775776346.460472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000100:00000001:0.0:1775776346.460473:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776346.460479:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776346.460481:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776346.460484:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776346.460486:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776346.460503:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776346.460506:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776346.460510:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460512:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460513:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460516:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776346.460518:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776346.460523:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460527:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776346.460530:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776346.460535:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776346.460540:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460544:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776346.460547:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776346.460552:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460556:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776346.460561:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776346.460567:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776346.460575:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776346.460577:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460580:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776346.460582:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776346.460585:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776346.460592:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776346.460598:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776346.460609:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099739648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776346.460622:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776346.460626:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776346.460633:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099739648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776346.460653:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776346.460656:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776346.460659:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776346.460660:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776346.460661:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776346.460663:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776346.460666:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18061 00000400:00000200:0.0:1775776346.460670:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776346.460672:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776346.460674:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776346.460676:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460678:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460680:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460681:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776346.460684:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776346.460686:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776346.460687:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776346.460690:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776346.460694:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776346.460698:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776346.460702:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460705:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460708:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460713:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776346.460716:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776346.460723:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460726:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776346.460728:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776346.460733:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776346.460735:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460736:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776346.460739:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776346.460742:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460744:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776346.460750:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776346.460754:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776346.460759:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776346.460761:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.460764:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776346.460766:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776346.460769:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776346.460775:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776346.460879:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776346.460888:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099739776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776346.460901:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776346.460904:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776346.460910:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099739776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776346.460921:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776346.460925:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776346.460927:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776346.460929:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776346.460932:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776346.460936:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776346.460939:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18061 00000400:00000200:0.0:1775776346.460942:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776346.460947:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776346.460950:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776346.460955:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460959:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460963:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460969:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776346.460974:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776346.460977:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776346.460980:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776346.460982:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776346.460986:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776346.460990:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776346.460993:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.460995:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776346.460999:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776346.461004:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776346.461008:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776346.461013:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776346.461017:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776346.461019:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776346.461023:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776346.461027:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.461030:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776346.461033:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776346.461036:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.461040:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776346.461044:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776346.461048:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776346.461051:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776346.461054:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776346.461058:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776346.461061:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776346.461064:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776346.461070:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776346.461073:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776346.461078:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099739904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776346.461104:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776346.461108:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776346.461115:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099739904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776346.461133:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776346.461139:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18071) 00000800:00000200:3.1:1775776346.461164:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776346.461166:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775776346.461405:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776346.461407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775776346.461645:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776346.461647:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775776346.461968:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776346.461971:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000100:00000001:0.0:1775776347.163893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776347.163899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776347.163901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776347.163905:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776347.163908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776347.163911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776347.163913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776347.163914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776347.163916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776347.227948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776347.227950:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776347.227952:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776347.739885:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312735477, at 4312735488 00000100:00000040:1.0:1775776347.739887:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312735477, at 4312735488 00000100:00000010:1.0:1775776347.739898:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344a80. 00000100:00000010:0.0:1775776347.739901:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000100:00000001:0.0:1775776348.187870:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776348.187875:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776348.187876:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776348.187880:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776348.187882:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776348.187886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776348.187888:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776348.187889:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776348.187891:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776348.251987:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776348.251994:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776348.251997:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776349.211849:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776349.211855:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776349.211856:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776349.211860:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776349.211863:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776349.211866:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776349.211868:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776349.211870:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776349.211872:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776349.276000:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776349.276003:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776349.276005:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776350.235857:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776350.235862:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776350.235864:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776350.235868:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776350.235871:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776350.235874:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776350.235876:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776350.235877:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776350.235879:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776350.299975:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776350.299979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776350.299982:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776351.259944:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776351.259951:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776351.259953:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776351.259959:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776351.259963:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776351.259966:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776351.259969:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776351.259970:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776351.259972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776351.323969:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776351.323972:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776351.323975:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776351.580414:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776351.580417:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776351.580861:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776351.580865:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776351.581185:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776351.581187:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776351.581428:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776351.581430:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776351.581584:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776351.581585:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1775776352.283896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776352.283903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776352.283905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776352.283910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776352.283913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776352.283917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776352.283919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776352.283922:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776352.283925:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776352.347917:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776352.347921:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776352.347924:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776352.859871:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312740597, at 4312740608 00000100:00000040:1.0:1775776352.859871:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312740597, at 4312740608 00000100:00000010:1.0:1775776352.859884:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000010:0.0:1775776352.859887:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000001:0.0:1775776353.307911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776353.307917:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776353.307919:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776353.307924:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776353.307928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776353.307931:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776353.307933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776353.307936:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776353.307938:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776353.371985:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776353.371989:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776353.371992:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776354.331923:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776354.331929:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776354.331931:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776354.331936:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776354.331939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776354.331942:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776354.331945:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776354.331947:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776354.331949:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776354.395965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776354.395969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776354.395973:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776355.355903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776355.355910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776355.355912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776355.355916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776355.355920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776355.355922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776355.355925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776355.355927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776355.355930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776355.419954:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776355.419958:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776355.419961:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776356.379883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776356.379888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776356.379889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776356.379894:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776356.379897:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776356.379901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776356.379903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776356.379905:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776356.379908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776356.443971:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776356.443974:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776356.443976:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776356.699920:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776356.699928:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776356.699930:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776356.699934:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776356.699938:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776356.699945:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18072 00000400:00000200:0.0:1775776356.699961:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776356.699969:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776356.699973:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776356.699976:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.699979:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.699981:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.699983:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776356.699986:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776356.699988:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776356.699989:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776356.699992:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776356.699993:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776356.699994:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776356.699996:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.699997:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.699998:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.699999:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776356.700001:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776356.700003:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700004:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776356.700006:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776356.700009:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776356.700010:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700012:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776356.700013:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776356.700021:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776356.700023:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776356.700025:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776356.700027:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700028:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776356.700030:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776356.700033:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776356.700039:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776356.700044:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776356.700057:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099740672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776356.700064:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776356.700068:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776356.700071:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099740672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776356.700093:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776356.700096:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776356.700099:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776356.700100:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776356.700102:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776356.700105:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776356.700108:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18072 00000400:00000200:0.0:1775776356.700112:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776356.700115:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776356.700117:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776356.700120:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700122:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.700125:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700127:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776356.700129:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776356.700131:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776356.700132:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776356.700133:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776356.700135:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776356.700137:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776356.700139:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700140:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.700142:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700146:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776356.700148:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776356.700152:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700155:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776356.700157:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776356.700160:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776356.700162:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700163:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776356.700166:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776356.700169:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700171:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776356.700176:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776356.700179:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776356.700184:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776356.700185:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700189:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776356.700191:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776356.700193:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776356.700197:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776356.700201:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776356.700206:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099740800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776356.700218:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776356.700221:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776356.700225:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099740800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776356.700234:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776356.700236:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776356.700240:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776356.700241:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776356.700243:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776356.700245:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776356.700249:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18072 00000400:00000200:0.0:1775776356.700251:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776356.700254:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776356.700257:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776356.700262:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700264:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.700266:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700269:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776356.700272:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776356.700272:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776356.700273:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776356.700275:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776356.700278:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776356.700280:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776356.700282:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700283:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.700284:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700288:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776356.700290:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776356.700291:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700293:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776356.700295:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776356.700297:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776356.700299:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700302:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776356.700304:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776356.700306:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700308:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776356.700322:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776356.700325:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776356.700328:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776356.700330:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700333:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776356.700335:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776356.700338:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776356.700342:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776356.700346:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776356.700353:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099740928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776356.700367:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776356.700369:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776356.700377:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099740928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776356.700390:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776356.700394:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776356.700399:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776356.700402:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776356.700405:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776356.700407:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776356.700412:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18072 00000400:00000200:0.0:1775776356.700417:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776356.700423:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776356.700425:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776356.700430:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700432:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.700433:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700434:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776356.700436:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776356.700437:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776356.700438:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776356.700439:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776356.700441:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776356.700443:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776356.700449:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700451:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.700455:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700458:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776356.700460:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776356.700462:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700463:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776356.700466:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776356.700469:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776356.700472:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700475:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776356.700479:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776356.700481:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700484:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776356.700488:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776356.700491:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776356.700492:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776356.700495:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700496:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776356.700497:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776356.700498:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776356.700504:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776356.700507:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776356.700514:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099741056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775776356.700516:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776356.700522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1775776356.700531:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776356.700536:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776356.700543:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099741056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776356.700559:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776356.700561:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776356.700567:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776356.700569:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776356.700570:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776356.700574:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776356.700577:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18072 00000400:00000200:0.0:1775776356.700582:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776356.700587:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776356.700590:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776356.700594:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700597:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.700599:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700600:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776356.700601:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776356.700602:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776356.700603:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776356.700604:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776356.700606:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776356.700608:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776356.700612:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700614:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776356.700617:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700620:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776356.700624:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776356.700628:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776356.700630:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776356.700635:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776356.700637:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776356.700641:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700644:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776356.700647:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776356.700649:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700652:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776356.700656:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776356.700657:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776356.700659:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776356.700660:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776356.700664:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776356.700665:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776356.700667:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776356.700670:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776356.700674:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776356.700678:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099741184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776356.700686:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776356.700688:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776356.700690:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099741184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776356.700694:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776356.700699:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18082) 00000800:00000200:2.1:1775776356.701010:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776356.701023:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776356.701237:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776356.701239:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776356.701415:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776356.701417:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776356.701622:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776356.701625:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1775776357.404101:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776357.404111:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776357.404112:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776357.404120:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:0.0:1775776357.404124:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776357.404128:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776357.404131:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776357.404133:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776357.404136:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776357.467962:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776357.467965:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776357.467967:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776357.979901:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312745717, at 4312745728 00000100:00000040:1.0:1775776357.979908:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312745717, at 4312745728 00000100:00000010:0.0:1775776357.979921:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000010:1.0:1775776357.979927:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776358.492001:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776358.492005:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776358.492009:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776359.516003:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776359.516008:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776359.516011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776360.540075:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776360.540079:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776360.540081:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776361.563952:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776361.563956:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776361.563960:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776361.819910:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776361.819921:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776361.819924:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776361.819929:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776361.819934:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776361.819943:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18077 00000400:00000200:0.0:1775776361.819954:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776361.819961:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776361.819964:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776361.819975:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.819979:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.819981:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.819984:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776361.819989:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776361.819991:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776361.819994:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776361.819997:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776361.819999:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776361.820002:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776361.820005:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820007:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820009:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820011:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776361.820014:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776361.820017:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820020:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776361.820023:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776361.820027:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776361.820031:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820034:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776361.820037:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776361.820048:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776361.820052:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776361.820056:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776361.820058:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820062:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776361.820065:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776361.820070:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776361.820079:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776361.820085:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776361.820094:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099741312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776361.820102:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776361.820106:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776361.820109:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099741312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776361.820130:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776361.820133:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776361.820138:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776361.820139:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776361.820140:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776361.820143:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776361.820144:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18077 00000400:00000200:0.0:1775776361.820147:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776361.820150:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776361.820151:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776361.820154:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820156:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820157:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820159:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776361.820161:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776361.820162:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776361.820163:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776361.820165:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776361.820168:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776361.820170:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776361.820174:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820177:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820182:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820187:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776361.820193:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776361.820198:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820202:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776361.820204:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776361.820212:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776361.820216:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820233:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776361.820239:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776361.820244:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820247:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776361.820251:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775776361.820255:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776361.820259:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776361.820260:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820264:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776361.820267:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776361.820269:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776361.820276:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776361.820281:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776361.820289:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099741440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776361.820302:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776361.820307:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776361.820314:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099741440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776361.820335:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776361.820339:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776361.820346:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776361.820348:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776361.820352:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776361.820356:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776361.820362:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18077 00000400:00000200:0.0:1775776361.820373:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776361.820381:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776361.820384:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776361.820392:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820396:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820400:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820403:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776361.820407:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776361.820409:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776361.820412:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776361.820416:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776361.820420:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000800:00000200:2.1:1775776361.820424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:0.0:1775776361.820426:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000800:00000200:2.1:1775776361.820428:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000040:0.0:1775776361.820432:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820437:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820441:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820444:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776361.820449:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776361.820455:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820461:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776361.820473:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776361.820478:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776361.820486:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820490:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776361.820492:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776361.820497:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820500:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776361.820506:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776361.820511:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776361.820515:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776361.820517:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820531:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776361.820533:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776361.820536:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776361.820543:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776361.820547:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776361.820556:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099741568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776361.820570:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776361.820574:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776361.820580:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099741568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776361.820603:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776361.820607:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776361.820611:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776361.820614:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776361.820617:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776361.820632:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776361.820637:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18077 00000400:00000200:0.0:1775776361.820645:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776361.820652:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776361.820656:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776361.820661:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820665:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820669:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820673:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776361.820678:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776361.820681:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776361.820685:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776361.820687:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776361.820691:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776361.820695:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776361.820701:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820705:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820710:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820714:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776361.820718:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776361.820722:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820726:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776361.820729:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776361.820733:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776361.820737:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820741:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776361.820744:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776361.820748:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820752:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776361.820757:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776361.820760:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776361.820765:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776361.820768:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820772:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776361.820775:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776361.820813:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776361.820821:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776361.820826:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776361.820835:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099741696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776361.820847:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776361.820851:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776361.820857:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099741696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776361.820871:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776361.820875:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776361.820881:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776361.820884:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776361.820886:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776361.820889:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776361.820893:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18077 00000400:00000200:0.0:1775776361.820900:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776361.820906:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776361.820909:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776361.820912:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820917:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820921:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820926:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776361.820930:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776361.820932:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776361.820934:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776361.820937:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776361.820941:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776361.820946:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776361.820953:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820956:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776361.820961:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820964:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776361.820967:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776361.820972:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776361.820976:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776361.820978:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776361.820983:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776361.820987:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.820990:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776361.820993:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776361.820997:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.821001:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776361.821007:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776361.821010:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776361.821013:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776361.821015:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776361.821016:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776361.821019:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776361.821021:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776361.821026:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776361.821032:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776361.821040:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099741824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776361.821056:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776361.821059:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776361.821065:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099741824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776361.821084:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776361.821088:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18087) 00000800:00000200:2.1:1775776361.821097:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776361.821099:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776361.821318:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776361.821320:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776361.821487:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776361.821489:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776361.821701:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776361.821703:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775776362.587960:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776362.587965:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776362.587969:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776363.099906:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312750837, at 4312750848 00000100:00000040:1.0:1775776363.099906:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312750837, at 4312750848 00000100:00000010:1.0:1775776363.099923:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000010:0.0:1775776363.099926:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775776363.611965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776363.611969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776363.611973:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776364.636015:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776364.636019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776364.636022:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776365.659982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776365.659984:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776365.659987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776366.684117:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776366.684122:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776366.684125:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776366.940434:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776366.940437:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776366.941352:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776366.941382:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776366.941665:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776366.941667:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776366.941865:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776366.941876:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776366.942191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776366.942246:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775776367.708109:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776367.708112:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776367.708114:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776368.219856:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312755957, at 4312755968 00000100:00000040:1.0:1775776368.219856:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312755957, at 4312755968 00000100:00000010:1.0:1775776368.219870:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000010:0.0:1775776368.219873:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775776368.731976:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776368.731979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776368.731983:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776369.755978:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776369.755982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776369.755985:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776370.779956:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776370.779959:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776370.779961:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776371.804020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776371.804024:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776371.804028:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776372.059947:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776372.059960:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776372.059963:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776372.059968:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776372.059974:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776372.059982:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18087 00000400:00000200:0.0:1775776372.059994:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776372.060002:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776372.060005:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776372.060009:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060013:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060015:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060017:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776372.060022:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776372.060024:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776372.060028:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776372.060031:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776372.060033:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776372.060036:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776372.060039:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060053:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060056:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060058:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776372.060061:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776372.060065:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060068:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776372.060072:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776372.060076:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776372.060080:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060083:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776372.060086:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776372.060100:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776372.060105:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776372.060111:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776372.060113:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060116:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776372.060120:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776372.060125:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776372.060135:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776372.060143:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776372.060154:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099742592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776372.060166:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776372.060172:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776372.060177:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099742592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776372.060205:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776372.060209:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776372.060212:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776372.060214:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776372.060215:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776372.060219:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776372.060221:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18087 00000400:00000200:0.0:1775776372.060226:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776372.060230:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776372.060233:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776372.060237:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060240:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060245:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060249:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776372.060253:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776372.060255:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776372.060259:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776372.060262:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776372.060267:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776372.060272:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776372.060279:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060284:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060290:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060295:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776372.060299:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776372.060305:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060309:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776372.060311:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776372.060319:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776372.060325:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060328:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776372.060334:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776372.060339:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060342:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776372.060347:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776372.060353:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776372.060358:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776372.060361:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060366:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776372.060370:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776372.060374:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776372.060382:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776372.060387:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776372.060394:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099742720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776372.060409:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776372.060413:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776372.060419:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099742720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776372.060438:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776372.060442:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776372.060446:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776372.060448:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776372.060452:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776372.060458:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776372.060463:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18087 00000400:00000200:0.0:1775776372.060470:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776372.060475:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776372.060478:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776372.060481:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060484:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060486:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060491:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776372.060496:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776372.060499:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776372.060502:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776372.060504:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000800:00000200:2.1:1775776372.060507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:0.0:1775776372.060508:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000800:00000200:2.1:1775776372.060511:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:0.0:1775776372.060512:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776372.060518:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060522:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060527:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060530:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776372.060533:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776372.060538:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060543:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776372.060547:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776372.060550:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776372.060553:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060557:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776372.060561:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776372.060565:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060568:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776372.060571:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776372.060576:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776372.060581:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776372.060583:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060586:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776372.060588:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776372.060591:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776372.060598:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776372.060602:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776372.060610:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099742848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776372.060624:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776372.060637:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776372.060641:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099742848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776372.060650:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776372.060653:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776372.060657:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776372.060658:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776372.060661:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776372.060664:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776372.060669:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18087 00000400:00000200:0.0:1775776372.060678:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776372.060683:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776372.060686:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776372.060694:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060698:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060700:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060703:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776372.060706:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776372.060709:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776372.060713:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776372.060716:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776372.060721:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776372.060725:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776372.060732:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060736:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060741:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060747:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776372.060752:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776372.060757:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060762:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776372.060766:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776372.060771:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776372.060776:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060801:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776372.060804:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776372.060807:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060809:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776372.060814:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776372.060817:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776372.060819:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776372.060821:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.060826:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776372.060830:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776372.060832:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776372.060839:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776372.060845:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776372.060851:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099742976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776372.060862:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776372.060866:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776372.060871:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099742976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776372.060885:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776372.060889:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776372.060896:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776372.060898:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776372.060901:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776372.060904:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776372.060909:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18087 00000400:00000200:0.0:1775776372.060916:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776372.060921:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776372.060924:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776372.060932:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060936:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060940:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060943:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776372.060947:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776372.060949:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776372.060952:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776372.060956:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776372.060960:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776372.060965:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776372.060971:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060975:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776372.060979:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060983:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776372.060987:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776372.060990:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776372.060992:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776372.060995:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776372.060998:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776372.061003:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.061007:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776372.061010:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776372.061016:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.061019:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776372.061025:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776372.061030:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776372.061035:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776372.061038:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776372.061053:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776372.061056:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776372.061059:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776372.061065:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776372.061071:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776372.061077:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099743104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776372.061090:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776372.061094:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776372.061100:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099743104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776372.061130:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776372.061137:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18097) 00000800:00000200:2.1:1775776372.061191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776372.061194:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776372.061393:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776372.061396:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776372.061630:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776372.061632:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776372.061889:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776372.061890:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1775776372.828005:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776372.828009:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776372.828011:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776373.339867:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312761077, at 4312761088 00000100:00000010:0.0:1775776373.339889:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000040:1.0:1775776373.339895:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312761077, at 4312761088 00000100:00000010:1.0:1775776373.339910:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775776373.852026:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776373.852028:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776373.852031:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776374.876024:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776374.876028:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776374.876030:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776375.900017:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776375.900020:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776375.900023:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776376.924035:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776376.924039:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776376.924042:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776377.180609:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776377.181157:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776377.181463:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776377.181616:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776377.181746:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775776377.948037:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776377.948040:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776377.948042:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776378.459892:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312766197, at 4312766208 00000100:00000040:1.0:1775776378.459893:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312766197, at 4312766208 00000100:00000010:1.0:1775776378.459906:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000010:0.0:1775776378.459911:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775776378.972035:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776378.972038:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776378.972041:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776379.996108:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776379.996111:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776379.996114:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775776380.252162:0:301879:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:0.0:1775776380.252201:0:301879:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775776380.252211:0:301879:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776380.252215:0:301879:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:0.0:1775776380.252220:0:301879:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776380.252224:0:301879:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775776380.252230:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776380.252233:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:0.0:1775776380.252239:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776380.252244:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776380.252249:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d4c8 00000400:00000010:0.0:1775776380.252252:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d4c8. 00000100:00000001:0.0:1775776380.252256:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776380.252264:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45defb80 x1862033099743232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.252291:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776380.252295:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45defb80 x1862033099743232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.252303:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.252305:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776380.252314:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776380.252316:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7500. 00000400:00000200:0.0:1775776380.252319:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776380.252322:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776380.252324:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d000 00000400:00000010:0.0:1775776380.252326:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d000. 00000100:00000001:0.0:1775776380.252329:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776380.252334:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45deca80 x1862033099743360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.254179:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776380.254189:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deca80 x1862033099743360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.254198:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.254200:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776380.254215:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776380.254218:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7100. 00000400:00000200:0.0:1775776380.254222:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776380.254227:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776380.254232:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720de58 00000400:00000010:0.0:1775776380.254235:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720de58. 00000100:00000001:0.0:1775776380.254238:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776380.254242:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dec380 x1862033099743488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.254251:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776380.254253:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec380 x1862033099743488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.254259:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.254260:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776380.254264:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776380.254266:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7600. 00000400:00000200:0.0:1775776380.254269:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776380.254272:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776380.254274:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dcc0 00000400:00000010:0.0:1775776380.254276:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dcc0. 00000100:00000001:0.0:1775776380.254278:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776380.254281:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dece00 x1862033099743616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.254293:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776380.254295:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dece00 x1862033099743616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.254301:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.254303:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776380.254308:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776380.254311:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:0.0:1775776380.254313:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776380.254316:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776380.254319:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dc38 00000400:00000010:0.0:1775776380.254321:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dc38. 00000100:00000001:0.0:1775776380.254323:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776380.254328:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dee680 x1862033099743744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.254335:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776380.254337:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee680 x1862033099743744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776402 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.254343:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.254344:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775776381.020010:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776381.020013:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776381.020015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776382.044006:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776382.044009:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776382.044012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776382.300523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776382.300528:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776382.301164:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776382.301166:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776382.301426:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776382.301427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776382.301599:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776382.301615:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776382.301968:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776382.301970:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775776383.067973:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776383.067976:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776383.067979:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776383.579891:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312771317, at 4312771328 00000100:00000040:1.0:1775776383.579891:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312771317, at 4312771328 00000100:00000010:0.0:1775776383.579909:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000010:1.0:1775776383.579911:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776384.092021:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776384.092026:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776384.092029:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776385.115959:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776385.115962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776385.115965:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776386.140019:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776386.140023:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776386.140026:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776387.163934:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776387.163938:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776387.163941:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776387.420466:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776387.420472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776387.421151:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776387.421153:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776387.421412:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776387.421414:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776387.421664:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776387.421666:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776387.422050:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776387.422053:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775776388.187915:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776388.187919:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776388.187922:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776388.699885:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312776437, at 4312776448 00000100:00000040:1.0:1775776388.699885:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312776437, at 4312776448 00000100:00000010:1.0:1775776388.699900:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000010:0.0:1775776388.699905:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000020:00000001:1.0:1775776389.211974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776389.211977:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776389.211980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776390.235963:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776390.235966:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776390.235968:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776391.259972:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776391.259975:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776391.259978:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776392.283957:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776392.283960:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776392.283962:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776392.539891:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776392.539901:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776392.539904:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776392.539908:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776392.539912:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776392.539918:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18107 00000400:00000200:0.0:1775776392.539928:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776392.539935:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776392.539938:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776392.539941:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.539944:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.539946:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.539949:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776392.539953:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776392.539954:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776392.539957:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776392.539960:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776392.539962:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776392.539964:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776392.539967:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.539969:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.539971:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.539973:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776392.539976:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776392.539979:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776392.539982:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776392.539984:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776392.539987:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776392.539990:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.539992:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776392.539994:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776392.540003:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776392.540006:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776392.540009:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776392.540011:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540013:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776392.540016:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776392.540020:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776392.540027:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776392.540034:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776392.540043:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099745152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776392.540053:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776392.540057:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776392.540062:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099745152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776392.540082:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776392.540085:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776392.540089:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776392.540096:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776392.540098:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776392.540104:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776392.540107:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18107 00000400:00000200:0.0:1775776392.540113:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776392.540119:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776392.540123:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776392.540130:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540135:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.540139:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540142:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776392.540147:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776392.540149:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776392.540152:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776392.540156:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776392.540160:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776392.540163:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776392.540169:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540173:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.540177:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540180:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776392.540184:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776392.540188:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540192:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776392.540195:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776392.540200:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776392.540203:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540205:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776392.540210:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776392.540214:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540219:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776392.540225:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776392.540229:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776392.540233:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776392.540235:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540238:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776392.540241:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776392.540244:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776392.540251:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776392.540256:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776392.540266:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099745280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776392.540280:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776392.540284:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776392.540300:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099745280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776392.540323:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776392.540326:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776392.540334:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776392.540337:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776392.540341:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776392.540347:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776392.540352:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18107 00000400:00000200:0.0:1775776392.540358:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776392.540364:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776392.540369:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776392.540373:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540377:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775776392.540380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:0.0:1775776392.540382:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776392.540385:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1775776392.540386:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776392.540390:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776392.540392:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776392.540395:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776392.540398:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776392.540401:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776392.540405:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776392.540410:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540415:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.540420:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540424:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776392.540429:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776392.540433:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540438:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776392.540442:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776392.540446:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776392.540450:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540452:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776392.540455:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776392.540467:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540472:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776392.540476:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776392.540481:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776392.540485:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776392.540487:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540490:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776392.540491:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776392.540493:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776392.540497:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776392.540500:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776392.540508:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099745408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776392.540519:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776392.540523:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776392.540531:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099745408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776392.540543:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776392.540548:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776392.540554:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776392.540557:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776392.540560:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776392.540564:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776392.540569:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18107 00000400:00000200:0.0:1775776392.540577:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776392.540583:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776392.540589:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776392.540595:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540600:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.540604:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540609:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776392.540614:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776392.540618:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776392.540621:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776392.540625:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776392.540628:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776392.540634:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776392.540642:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540648:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.540653:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540658:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776392.540663:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776392.540670:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540675:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776392.540679:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776392.540683:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776392.540702:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540708:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776392.540710:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776392.540716:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540721:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776392.540726:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776392.540731:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776392.540734:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776392.540735:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540736:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776392.540738:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776392.540741:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776392.540747:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776392.540753:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776392.540762:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099745536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776392.540820:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776392.540825:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776392.540833:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099745536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776392.540849:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776392.540853:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776392.540859:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776392.540862:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776392.540864:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776392.540870:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776392.540873:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18107 00000400:00000200:0.0:1775776392.540880:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776392.540886:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776392.540890:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776392.540895:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540899:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.540904:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540908:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776392.540911:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776392.540914:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776392.540917:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776392.540921:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776392.540925:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776392.540929:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776392.540935:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540939:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776392.540944:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540949:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776392.540952:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776392.540958:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776392.540964:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776392.540968:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776392.540973:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776392.540978:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540982:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776392.540986:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776392.540990:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.540994:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776392.541001:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776392.541006:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776392.541011:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776392.541015:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776392.541019:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776392.541021:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776392.541025:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776392.541032:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776392.541037:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776392.541046:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099745664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775776392.541051:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776392.541057:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1775776392.541062:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776392.541067:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776392.541074:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099745664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776392.541089:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776392.541095:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18117) 00000800:00000200:2.1:1775776392.541312:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776392.541315:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776392.541551:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776392.541552:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776392.541751:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776392.541753:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775776393.307990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776393.307993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776393.307997:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776393.819871:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312781557, at 4312781568 00000100:00000040:1.0:1775776393.819871:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312781557, at 4312781568 00000100:00000010:0.0:1775776393.819884:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000010:1.0:1775776393.819886:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000020:00000001:1.0:1775776394.331961:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776394.331964:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776394.331966:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776395.355962:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776395.355965:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776395.355969:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776396.379928:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776396.379931:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776396.379933:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776397.404001:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776397.404004:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776397.404008:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1775776397.660244:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776397.660258:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776397.660650:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776397.660652:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776397.660874:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776397.660876:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776397.661028:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776397.661029:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776397.661179:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776397.661180:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1775776398.427905:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776398.427907:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776398.427910:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776398.939853:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312786677, at 4312786688 00000100:00000010:1.0:1775776398.939864:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000040:0.0:1775776398.939917:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312786677, at 4312786688 00000100:00000010:0.0:1775776398.939922:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775776399.451993:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776399.451995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776399.451997:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776400.475959:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776400.475963:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776400.475968:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776401.499935:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776401.499937:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776401.499939:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776402.523942:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776402.523946:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776402.523948:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776402.779879:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776402.779886:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776402.779888:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776402.779891:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776402.779894:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776402.779900:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18118 00000400:00000200:1.0:1775776402.779908:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776402.779914:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776402.779917:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776402.779920:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.779923:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.779925:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.779927:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776402.779931:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776402.779933:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776402.779936:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776402.779939:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776402.779941:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776402.779943:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776402.779946:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.779948:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.779950:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.779952:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776402.779955:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776402.779958:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776402.779960:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776402.779962:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776402.779965:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776402.779966:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.779968:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776402.779969:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776402.779976:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776402.779979:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776402.779982:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776402.779983:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.779993:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776402.779996:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776402.779998:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776402.780004:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776402.780010:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776402.780016:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099746432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776402.780023:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776402.780026:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776402.780029:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099746432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776402.780047:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776402.780050:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776402.780053:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776402.780054:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776402.780055:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776402.780058:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776402.780061:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18118 00000400:00000200:1.0:1775776402.780065:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776402.780068:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776402.780071:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776402.780074:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780076:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.780079:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780088:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776402.780091:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776402.780094:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776402.780097:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776402.780100:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776402.780103:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776402.780108:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776402.780115:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780119:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.780121:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780124:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776402.780128:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776402.780130:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780133:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776402.780136:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776402.780142:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776402.780146:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780151:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776402.780161:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776402.780167:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780170:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776402.780177:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776402.780180:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776402.780186:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776402.780188:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780192:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776402.780193:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776402.780196:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776402.780202:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776402.780208:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776402.780218:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099746560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776402.780242:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776402.780245:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776402.780250:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099746560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776402.780262:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776402.780264:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776402.780269:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776402.780271:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776402.780272:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776402.780276:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776402.780282:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18118 00000400:00000200:1.0:1775776402.780286:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776402.780291:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776402.780293:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776402.780298:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780301:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.780310:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780315:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776402.780324:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776402.780326:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776402.780329:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776402.780333:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776402.780337:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776402.780350:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776402.780358:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780362:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.780368:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780371:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776402.780375:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000800:00000200:2.1:1775776402.780377:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:1.0:1775776402.780381:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000800:00000200:2.1:1775776402.780383:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000001:1.0:1775776402.780386:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776402.780390:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776402.780396:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776402.780399:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780401:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776402.780405:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776402.780410:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780418:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776402.780426:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776402.780432:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776402.780437:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776402.780439:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780442:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776402.780444:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776402.780448:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776402.780454:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776402.780460:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776402.780470:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099746688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776402.780487:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776402.780490:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776402.780493:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099746688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776402.780500:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776402.780503:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776402.780518:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776402.780521:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776402.780523:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776402.780528:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776402.780530:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18118 00000400:00000200:1.0:1775776402.780537:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776402.780541:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776402.780543:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776402.780545:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780546:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.780547:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780550:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776402.780554:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776402.780559:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776402.780563:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776402.780566:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776402.780570:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776402.780575:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776402.780582:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780586:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.780592:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780597:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776402.780602:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776402.780607:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780609:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776402.780618:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776402.780621:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776402.780627:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780628:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776402.780630:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776402.780633:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780635:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776402.780640:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776402.780643:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776402.780646:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776402.780648:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780650:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776402.780652:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776402.780654:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776402.780659:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776402.780664:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776402.780672:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099746816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776402.780688:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776402.780691:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776402.780696:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099746816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776402.780709:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776402.780713:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776402.780718:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776402.780720:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776402.780723:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776402.780725:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776402.780729:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18118 00000400:00000200:1.0:1775776402.780734:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776402.780738:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776402.780742:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776402.780746:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780750:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.780752:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780753:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776402.780755:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776402.780757:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776402.780760:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776402.780762:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776402.780764:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776402.780768:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776402.780774:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780776:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776402.780812:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780815:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776402.780820:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776402.780825:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776402.780829:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776402.780831:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776402.780834:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776402.780838:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780839:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776402.780842:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776402.780845:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780847:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776402.780850:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776402.780853:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776402.780856:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776402.780857:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776402.780859:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776402.780860:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776402.780861:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776402.780865:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776402.780868:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776402.780873:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099746944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776402.780882:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776402.780883:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776402.780888:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099746944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776402.780899:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776402.780902:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18128) 00000800:00000200:2.1:1775776402.781016:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776402.781019:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776402.781278:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776402.781280:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776402.781556:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776402.781558:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776402.781919:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776402.781922:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1775776403.547959:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776403.547963:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776403.547966:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776404.059878:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312791797, at 4312791808 00000100:00000040:0.0:1775776404.059894:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312791797, at 4312791808 00000100:00000010:1.0:1775776404.059894:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000010:0.0:1775776404.059902:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775776404.572014:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776404.572017:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776404.572021:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776405.595985:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776405.595989:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776405.595993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776406.619979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776406.619982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776406.619984:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776407.643946:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776407.643950:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776407.643954:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776407.899900:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776407.899909:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776407.899912:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776407.899915:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776407.899918:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776407.899924:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18123 00000400:00000200:0.0:1775776407.899931:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776407.899936:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776407.899938:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776407.899940:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.899942:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.899944:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.899945:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776407.899948:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776407.899949:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776407.899951:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776407.899957:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776407.899958:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776407.899959:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776407.899961:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.899962:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.899963:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.899965:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776407.899966:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776407.899968:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776407.899970:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776407.899972:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776407.899973:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776407.899975:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.899977:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776407.899978:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776407.899985:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776407.899988:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776407.899991:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776407.899992:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.899994:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776407.899996:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776407.899999:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776407.900006:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776407.900010:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776407.900017:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099747072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776407.900025:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776407.900028:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776407.900031:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099747072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776407.900047:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776407.900049:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776407.900051:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776407.900052:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776407.900053:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776407.900054:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776407.900056:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18123 00000400:00000200:0.0:1775776407.900058:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776407.900060:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776407.900061:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776407.900063:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900064:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.900065:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900066:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776407.900068:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776407.900069:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776407.900069:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776407.900071:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776407.900073:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776407.900075:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776407.900077:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900078:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.900079:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900080:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776407.900081:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776407.900084:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900086:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776407.900088:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776407.900093:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776407.900095:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900097:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776407.900102:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776407.900106:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900109:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776407.900112:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776407.900115:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776407.900119:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776407.900120:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900121:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776407.900124:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776407.900125:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776407.900129:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776407.900131:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776407.900144:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099747200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776407.900151:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776407.900153:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776407.900155:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099747200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776407.900168:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776407.900170:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776407.900172:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776407.900173:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776407.900175:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776407.900179:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776407.900181:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18123 00000400:00000200:0.0:1775776407.900188:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776407.900191:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776407.900194:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776407.900196:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900197:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.900200:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900201:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776407.900203:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776407.900203:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776407.900204:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776407.900206:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776407.900209:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776407.900212:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776407.900216:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900218:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.900219:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900221:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776407.900223:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776407.900226:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900230:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776407.900233:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776407.900236:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776407.900240:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900243:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776407.900246:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776407.900251:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900252:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776407.900257:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776407.900260:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776407.900263:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776407.900265:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900267:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776407.900268:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776407.900271:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776407.900278:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776407.900283:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776407.900291:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099747328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776407.900305:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776407.900310:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776407.900318:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099747328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776407.900336:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776407.900340:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776407.900344:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776407.900345:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776407.900346:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776407.900349:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776407.900351:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18123 00000400:00000200:0.0:1775776407.900358:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776407.900363:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776407.900364:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1775776407.900367:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:0.0:1775776407.900370:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776407.900371:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1775776407.900372:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.900376:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900379:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776407.900381:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776407.900383:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776407.900385:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776407.900386:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776407.900396:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776407.900400:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776407.900404:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900406:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.900410:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900413:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776407.900439:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776407.900446:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900452:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776407.900456:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776407.900459:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776407.900464:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900468:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776407.900471:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776407.900475:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900479:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776407.900484:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776407.900486:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776407.900489:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776407.900491:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900494:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776407.900496:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776407.900500:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776407.900504:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776407.900508:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776407.900514:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099747456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776407.900523:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776407.900525:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776407.900531:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099747456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776407.900542:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776407.900545:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776407.900552:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776407.900555:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776407.900557:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776407.900560:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776407.900563:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18123 00000400:00000200:0.0:1775776407.900569:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776407.900575:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776407.900577:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776407.900582:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900587:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.900592:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900596:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776407.900602:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776407.900604:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776407.900608:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776407.900611:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776407.900616:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776407.900622:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776407.900631:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900636:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776407.900641:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900643:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776407.900648:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776407.900653:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776407.900658:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776407.900660:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776407.900664:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776407.900667:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900669:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776407.900671:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776407.900675:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900677:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776407.900683:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776407.900686:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776407.900691:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776407.900693:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776407.900697:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776407.900700:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776407.900702:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776407.900708:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776407.900712:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776407.900718:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099747584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776407.900731:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776407.900734:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776407.900739:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099747584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776407.900752:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776407.900757:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18133) 00000800:00000200:2.1:1775776407.900885:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776407.900886:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776407.901056:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776407.901067:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776407.901274:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776407.901276:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776407.901517:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776407.901518:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775776408.667982:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776408.667985:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776408.667988:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776409.179864:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312796917, at 4312796928 00000100:00000010:1.0:1775776409.179877:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000040:0.0:1775776409.179893:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312796917, at 4312796928 00000100:00000010:0.0:1775776409.179903:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000020:00000001:1.0:1775776409.691980:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776409.691984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776409.691987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776410.716042:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776410.716047:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776410.716051:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776411.740003:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776411.740006:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776411.740009:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776412.764007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776412.764016:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776412.764019:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776413.019942:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776413.019951:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776413.019953:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776413.019956:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776413.019960:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776413.019965:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18128 00000400:00000200:1.0:1775776413.019974:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776413.019979:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776413.019981:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776413.019984:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.019986:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.019988:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.019990:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776413.019993:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776413.019995:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776413.019997:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776413.019999:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776413.020001:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776413.020002:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776413.020005:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020007:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020009:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020011:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776413.020014:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776413.020017:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020020:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776413.020024:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776413.020027:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776413.020030:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020032:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776413.020035:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776413.020045:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776413.020058:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776413.020063:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776413.020065:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020068:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776413.020070:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776413.020074:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776413.020082:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775776413.020089:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776413.020098:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099747712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776413.020108:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776413.020113:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776413.020118:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099747712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776413.020152:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776413.020156:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776413.020159:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776413.020160:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776413.020161:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776413.020163:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776413.020165:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18128 00000400:00000200:1.0:1775776413.020169:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776413.020171:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776413.020173:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776413.020176:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020184:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020187:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020190:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776413.020194:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776413.020196:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776413.020199:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776413.020202:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776413.020207:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776413.020212:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776413.020220:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020226:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020230:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020233:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776413.020237:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776413.020246:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020251:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776413.020253:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776413.020260:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776413.020264:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020268:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776413.020274:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776413.020279:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020283:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776413.020288:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776413.020292:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776413.020299:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776413.020302:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020305:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776413.020309:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776413.020312:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776413.020319:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776413.020324:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776413.020348:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099747840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776413.020355:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776413.020356:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776413.020359:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099747840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776413.020369:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776413.020374:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776413.020377:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776413.020381:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776413.020385:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776413.020389:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776413.020393:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18128 00000400:00000200:1.0:1775776413.020401:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776413.020409:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776413.020413:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776413.020419:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020424:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020429:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020436:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776413.020441:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776413.020444:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776413.020447:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776413.020452:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776413.020455:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776413.020460:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776413.020468:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020472:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020478:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020483:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776413.020487:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000800:00000200:2.1:1775776413.020498:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:1.0:1775776413.020500:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020503:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776413.020503:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 02000000:00000001:1.0:1775776413.020507:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776413.020510:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776413.020513:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020515:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776413.020516:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776413.020518:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020520:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776413.020523:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776413.020526:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776413.020528:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776413.020530:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020532:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776413.020535:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776413.020537:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776413.020541:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776413.020546:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776413.020552:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099747968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776413.020569:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776413.020573:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776413.020578:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099747968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776413.020597:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776413.020601:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776413.020606:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776413.020609:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776413.020610:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776413.020613:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776413.020617:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18128 00000400:00000200:1.0:1775776413.020624:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776413.020627:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776413.020630:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776413.020632:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020633:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020635:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020638:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776413.020641:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776413.020642:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776413.020643:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776413.020645:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776413.020647:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776413.020650:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776413.020652:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020654:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020655:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020657:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776413.020660:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776413.020663:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020664:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776413.020667:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776413.020670:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776413.020672:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020676:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776413.020678:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776413.020680:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020683:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776413.020687:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776413.020689:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776413.020691:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776413.020692:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020693:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776413.020694:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776413.020694:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776413.020698:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776413.020702:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776413.020708:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099748096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776413.020726:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776413.020729:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776413.020735:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099748096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776413.020749:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776413.020754:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776413.020760:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776413.020763:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776413.020767:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776413.020774:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776413.020820:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18128 00000400:00000200:1.0:1775776413.020829:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776413.020836:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776413.020841:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776413.020848:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020852:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020857:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020859:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776413.020863:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776413.020864:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776413.020867:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776413.020869:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776413.020873:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776413.020878:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776413.020884:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020887:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776413.020891:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020893:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776413.020898:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776413.020902:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776413.020905:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776413.020908:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776413.020912:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776413.020915:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020917:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776413.020918:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776413.020920:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020921:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776413.020924:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776413.020927:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776413.020929:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776413.020930:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776413.020931:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776413.020932:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776413.020934:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776413.020937:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776413.020941:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776413.020948:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099748224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776413.020957:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776413.020960:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776413.020967:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099748224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776413.020980:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776413.020985:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18138) 00000800:00000200:2.1:1775776413.021159:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776413.021161:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776413.021417:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776413.021420:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776413.021623:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776413.021625:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776413.021891:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776413.021893:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1775776413.787952:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776413.787954:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776413.787957:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776414.299866:0:314402:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312802037, at 4312802048 00000100:00000010:1.0:1775776414.299883:0:314402:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000040:0.0:1775776414.299890:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312802037, at 4312802048 00000100:00000010:0.0:1775776414.299897:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000020:00000001:1.0:1775776414.811994:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776414.811997:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776414.812000:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776415.835984:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776415.835987:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776415.835990:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776416.859964:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776416.859967:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776416.859969:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776417.883928:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776417.883932:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776417.883935:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776418.139914:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776418.139930:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776418.139933:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776418.139937:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776418.139941:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776418.139948:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18133 00000400:00000200:0.0:1775776418.139957:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776418.139963:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776418.139966:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776418.139970:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.139973:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.139975:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.139978:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776418.139982:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776418.139983:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776418.139986:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776418.139988:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776418.139991:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776418.139993:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776418.139996:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.139998:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.139999:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140001:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776418.140004:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776418.140007:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140009:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776418.140011:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776418.140014:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776418.140017:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140019:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776418.140022:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776418.140030:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776418.140034:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776418.140037:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776418.140039:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140042:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776418.140044:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776418.140048:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776418.140055:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776418.140061:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776418.140069:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099748352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776418.140079:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776418.140083:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776418.140088:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099748352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776418.140122:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776418.140126:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776418.140129:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776418.140130:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776418.140132:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776418.140134:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776418.140137:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18133 00000400:00000200:0.0:1775776418.140141:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776418.140144:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776418.140147:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776418.140151:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140153:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.140155:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140158:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776418.140163:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776418.140164:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776418.140166:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776418.140169:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776418.140173:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776418.140177:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776418.140185:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140189:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.140194:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140198:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776418.140203:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776418.140209:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140213:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776418.140217:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776418.140222:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776418.140226:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140229:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776418.140233:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776418.140255:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140258:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776418.140262:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776418.140264:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776418.140268:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776418.140271:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140274:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776418.140276:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776418.140279:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776418.140285:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776418.140290:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776418.140298:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099748480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776418.140312:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776418.140316:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776418.140323:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099748480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776418.140340:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776418.140345:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776418.140351:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776418.140354:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776418.140356:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776418.140359:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776418.140372:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18133 00000400:00000200:0.0:1775776418.140379:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776418.140385:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776418.140389:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776418.140396:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140401:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.140406:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140410:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776418.140414:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776418.140415:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776418.140417:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776418.140420:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776418.140425:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776418.140430:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776418.140437:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140441:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.140443:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140447:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776418.140450:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776418.140456:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140461:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776418.140463:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1775776418.140464:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776418.140467:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775776418.140468:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 02000000:00000001:0.0:1775776418.140472:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140476:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776418.140479:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776418.140483:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140484:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776418.140490:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776418.140493:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776418.140496:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776418.140498:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140500:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776418.140502:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776418.140505:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776418.140510:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776418.140516:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776418.140523:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099748608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776418.140540:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776418.140544:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776418.140551:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099748608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776418.140563:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776418.140567:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776418.140574:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776418.140575:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776418.140577:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776418.140579:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776418.140583:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18133 00000400:00000200:0.0:1775776418.140591:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776418.140597:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776418.140601:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776418.140607:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140611:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.140613:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140617:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776418.140619:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776418.140620:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776418.140623:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776418.140625:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776418.140628:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776418.140631:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776418.140634:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140637:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.140640:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140641:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776418.140645:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776418.140649:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140653:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776418.140657:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776418.140659:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776418.140661:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140664:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776418.140665:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776418.140669:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140673:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776418.140677:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776418.140680:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776418.140683:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776418.140685:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140687:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776418.140690:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776418.140692:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776418.140695:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776418.140698:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776418.140705:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099748736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776418.140717:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776418.140720:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776418.140726:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099748736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776418.140747:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776418.140751:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776418.140759:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776418.140762:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776418.140765:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776418.140768:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776418.140774:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18133 00000400:00000200:0.0:1775776418.140808:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776418.140812:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776418.140817:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776418.140824:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140828:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.140830:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140834:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776418.140837:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776418.140839:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776418.140841:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776418.140843:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776418.140847:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776418.140851:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776418.140857:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140861:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776418.140864:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140867:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776418.140869:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776418.140873:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776418.140876:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776418.140880:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776418.140883:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776418.140885:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140886:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776418.140888:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776418.140891:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140895:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776418.140899:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776418.140903:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776418.140905:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776418.140906:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776418.140907:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776418.140908:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776418.140911:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776418.140914:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776418.140919:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776418.140925:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099748864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776418.140938:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776418.140941:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776418.140947:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099748864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776418.140961:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776418.140966:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18143) 00000800:00000200:2.1:1775776418.141062:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776418.141064:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776418.141347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776418.141350:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776418.141567:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776418.141568:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776418.141848:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776418.141860:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775776418.907986:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776418.907989:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776418.907991:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776419.419875:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312807157, at 4312807168 00000100:00000010:1.0:1775776419.419889:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000040:0.0:1775776419.419890:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312807157, at 4312807168 00000100:00000010:0.0:1775776419.419899:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775776419.931931:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776419.931935:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776419.931937:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776420.955960:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776420.955963:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776420.955967:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776421.979969:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776421.979972:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776421.979974:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776423.004011:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776423.004015:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776423.004017:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776423.259895:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776423.259904:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776423.259907:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776423.259910:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776423.259913:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776423.259918:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18138 00000400:00000200:1.0:1775776423.259925:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776423.259929:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776423.259931:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776423.259934:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.259936:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.259937:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.259939:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776423.259941:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776423.259943:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776423.259944:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776423.259947:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776423.259948:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776423.259950:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776423.259952:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.259953:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.259954:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.259955:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776423.259957:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776423.259959:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776423.259960:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776423.259962:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776423.259964:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776423.259966:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.259967:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776423.259969:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776423.259977:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776423.259979:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776423.259982:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776423.259983:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.259985:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776423.259987:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776423.259990:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776423.259996:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776423.260001:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776423.260007:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099748992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776423.260014:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776423.260017:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776423.260020:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099748992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776423.260036:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776423.260038:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776423.260040:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776423.260041:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776423.260042:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776423.260044:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776423.260045:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18138 00000400:00000200:1.0:1775776423.260047:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776423.260049:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776423.260051:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776423.260052:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260053:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.260054:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260056:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776423.260057:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776423.260059:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776423.260060:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776423.260061:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776423.260063:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776423.260064:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776423.260066:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260067:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.260069:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260070:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776423.260073:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776423.260075:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260078:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776423.260081:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776423.260084:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776423.260086:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260090:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776423.260094:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776423.260098:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260102:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776423.260104:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776423.260106:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776423.260110:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776423.260112:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260115:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776423.260116:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776423.260117:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776423.260119:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776423.260121:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776423.260128:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099749120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776423.260137:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776423.260140:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776423.260143:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099749120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776423.260176:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776423.260180:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776423.260182:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776423.260183:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776423.260185:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776423.260190:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776423.260192:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18138 00000400:00000200:1.0:1775776423.260195:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776423.260201:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776423.260205:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776423.260213:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260217:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.260224:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260228:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776423.260233:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776423.260242:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776423.260244:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776423.260246:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776423.260249:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776423.260252:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776423.260256:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260259:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.260262:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260265:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776423.260268:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776423.260272:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260275:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776423.260278:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776423.260281:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776423.260283:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260285:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776423.260287:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776423.260291:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260295:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776423.260299:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776423.260303:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776423.260305:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776423.260307:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260309:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776423.260310:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776423.260311:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776423.260316:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776423.260320:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776423.260323:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00100000:1.0:1775776423.260325:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099749248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775776423.260331:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1775776423.260332:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776423.260335:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776423.260343:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099749248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776423.260361:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776423.260366:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776423.260371:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776423.260375:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776423.260378:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776423.260383:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776423.260389:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18138 00000400:00000200:1.0:1775776423.260397:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776423.260404:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776423.260408:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776423.260414:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260418:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.260424:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260427:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776423.260431:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776423.260431:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776423.260433:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776423.260436:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776423.260438:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776423.260439:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776423.260442:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260445:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.260446:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260447:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776423.260450:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776423.260453:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260456:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776423.260458:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776423.260461:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776423.260463:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260466:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776423.260468:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776423.260470:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260472:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776423.260475:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776423.260479:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776423.260482:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776423.260484:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260487:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776423.260489:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776423.260491:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776423.260494:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776423.260499:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776423.260507:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099749376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776423.260514:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776423.260516:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776423.260518:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099749376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776423.260525:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776423.260528:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776423.260534:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776423.260536:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776423.260538:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776423.260541:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776423.260544:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18138 00000400:00000200:1.0:1775776423.260551:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776423.260555:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776423.260558:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776423.260561:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260565:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.260569:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260572:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776423.260576:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776423.260577:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776423.260579:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776423.260581:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776423.260584:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776423.260589:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776423.260595:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260598:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776423.260602:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260606:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776423.260609:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776423.260613:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776423.260625:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776423.260628:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776423.260630:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776423.260633:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260636:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776423.260637:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776423.260640:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260643:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776423.260645:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776423.260648:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776423.260655:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776423.260657:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776423.260660:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776423.260662:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776423.260664:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776423.260667:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776423.260671:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776423.260678:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099749504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776423.260697:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776423.260701:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776423.260711:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099749504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776423.260726:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776423.260733:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18148) 00000800:00000200:2.1:1775776423.260872:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776423.260875:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776423.261072:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776423.261074:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776423.261287:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776423.261288:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776423.261494:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776423.261495:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1775776424.027962:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776424.027966:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776424.027969:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776424.539941:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312812277, at 4312812288 00000100:00000040:1.0:1775776424.539941:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312812277, at 4312812288 00000100:00000010:0.0:1775776424.539956:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000010:1.0:1775776424.539958:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000020:00000001:1.0:1775776425.051979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776425.051982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776425.051984:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776426.076028:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776426.076033:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776426.076037:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776427.100016:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776427.100020:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776427.100023:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776428.124033:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776428.124035:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776428.124038:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776428.379915:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776428.379923:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776428.379925:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776428.379928:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776428.379931:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776428.379936:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18143 00000400:00000200:0.0:1775776428.379943:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776428.379948:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776428.379950:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776428.379952:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.379955:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.379956:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.379958:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776428.379960:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776428.379962:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776428.379964:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776428.379966:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776428.379967:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776428.379968:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776428.379976:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.379978:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.379979:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.379980:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776428.379982:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776428.379984:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776428.379986:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776428.379988:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776428.379991:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776428.379993:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.379994:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776428.379996:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776428.380003:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776428.380006:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776428.380009:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776428.380010:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380012:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776428.380014:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776428.380017:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776428.380023:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776428.380028:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776428.380034:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099749632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776428.380041:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776428.380044:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776428.380047:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099749632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776428.380080:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776428.380082:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776428.380085:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776428.380086:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776428.380087:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776428.380089:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776428.380095:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18143 00000400:00000200:0.0:1775776428.380101:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776428.380105:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776428.380108:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776428.380111:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380114:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.380118:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380121:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776428.380125:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776428.380127:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776428.380129:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776428.380131:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776428.380134:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776428.380137:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776428.380140:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380145:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.380148:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380151:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776428.380154:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776428.380157:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380160:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776428.380161:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776428.380166:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776428.380169:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380173:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776428.380177:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776428.380182:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380184:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776428.380187:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776428.380190:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776428.380191:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776428.380194:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380195:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776428.380196:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776428.380198:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776428.380202:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776428.380206:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776428.380213:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099749760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776428.380228:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776428.380232:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776428.380236:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099749760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776428.380246:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776428.380249:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776428.380253:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776428.380254:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776428.380255:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776428.380257:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776428.380260:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18143 00000400:00000200:0.0:1775776428.380266:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776428.380270:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776428.380272:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776428.380276:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380278:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.380280:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380282:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776428.380284:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776428.380286:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776428.380288:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776428.380291:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776428.380293:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776428.380297:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776428.380302:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380307:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.380313:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380317:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776428.380320:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776428.380327:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380334:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776428.380337:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776428.380340:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776428.380343:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380347:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776428.380350:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776428.380356:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776428.380359:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776428.380360:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776428.380363:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000010:0.0:1775776428.380365:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776428.380369:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776428.380372:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776428.380375:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380376:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776428.380377:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776428.380379:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776428.380385:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776428.380388:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776428.380398:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099749888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776428.380424:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776428.380429:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776428.380437:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099749888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776428.380451:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776428.380453:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776428.380456:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776428.380458:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776428.380460:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776428.380463:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776428.380468:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18143 00000400:00000200:0.0:1775776428.380474:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776428.380476:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776428.380479:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776428.380481:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380482:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.380484:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380488:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776428.380489:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776428.380490:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776428.380492:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776428.380493:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776428.380496:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776428.380499:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776428.380504:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380505:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.380509:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380511:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776428.380514:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776428.380518:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380521:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776428.380524:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776428.380526:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776428.380528:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380532:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776428.380534:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776428.380538:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380541:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776428.380546:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776428.380548:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776428.380550:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776428.380552:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380556:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776428.380558:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776428.380559:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776428.380564:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776428.380568:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776428.380574:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099750016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776428.380594:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776428.380596:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776428.380602:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099750016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776428.380625:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776428.380626:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776428.380630:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776428.380632:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776428.380634:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776428.380638:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776428.380640:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18143 00000400:00000200:0.0:1775776428.380642:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776428.380644:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776428.380645:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776428.380648:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380651:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.380653:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380656:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776428.380661:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776428.380663:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776428.380665:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776428.380667:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776428.380670:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776428.380673:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776428.380677:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380679:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776428.380683:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380689:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776428.380694:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776428.380699:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776428.380703:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776428.380706:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776428.380710:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776428.380716:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380721:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776428.380724:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776428.380728:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380730:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776428.380735:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776428.380739:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776428.380742:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776428.380744:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776428.380746:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776428.380748:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776428.380750:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776428.380755:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776428.380761:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776428.380767:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099750144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776428.380775:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776428.380824:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776428.380832:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099750144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776428.380846:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776428.380851:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18153) 00000800:00000200:2.1:1775776428.381000:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776428.381002:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776428.381282:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776428.381284:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776428.381545:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776428.381547:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776428.381726:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776428.381728:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775776429.147980:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776429.147985:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776429.147989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776429.659921:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312817397, at 4312817408 00000100:00000040:0.0:1775776429.659923:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312817397, at 4312817408 00000100:00000010:1.0:1775776429.659947:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000010:0.0:1775776429.659954:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775776430.171951:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776430.171954:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776430.171957:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776431.195976:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776431.195979:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776431.195981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776432.219967:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776432.219970:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776432.219974:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776433.243953:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776433.243957:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776433.243961:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776433.499927:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776433.499935:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776433.499937:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776433.499941:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776433.499945:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776433.499951:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18148 00000400:00000200:0.0:1775776433.499962:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776433.499997:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776433.500001:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776433.500005:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500009:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500012:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500015:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776433.500020:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776433.500022:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776433.500024:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776433.500027:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776433.500030:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776433.500032:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776433.500034:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500035:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500036:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500038:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776433.500040:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776433.500042:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500043:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776433.500045:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776433.500048:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776433.500050:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500051:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776433.500053:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776433.500060:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775776433.500063:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776433.500066:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776433.500067:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500069:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776433.500071:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776433.500074:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776433.500086:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776433.500091:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776433.500099:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099750272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776433.500106:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776433.500110:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776433.500114:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099750272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776433.500127:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776433.500129:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776433.500131:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776433.500132:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776433.500133:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776433.500135:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776433.500136:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18148 00000400:00000200:0.0:1775776433.500139:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776433.500141:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776433.500143:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776433.500145:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500147:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500149:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500150:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776433.500152:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776433.500153:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776433.500154:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776433.500155:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776433.500158:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776433.500160:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776433.500162:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500164:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500166:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500169:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776433.500170:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776433.500174:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500175:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776433.500176:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776433.500181:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776433.500184:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500188:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776433.500191:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776433.500195:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500198:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776433.500203:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776433.500214:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776433.500220:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776433.500222:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500225:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776433.500228:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776433.500231:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776433.500237:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776433.500241:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776433.500248:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099750400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776433.500258:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776433.500263:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776433.500268:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099750400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776433.500280:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776433.500283:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776433.500297:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776433.500299:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776433.500301:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776433.500307:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776433.500312:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18148 00000400:00000200:0.0:1775776433.500320:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776433.500326:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776433.500331:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776433.500333:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500336:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500339:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500341:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776433.500343:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776433.500345:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776433.500348:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776433.500351:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776433.500353:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776433.500355:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776433.500362:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500365:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500370:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500374:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776433.500378:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776433.500383:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500388:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776433.500391:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776433.500395:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776433.500399:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500402:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776433.500404:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776433.500407:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500409:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776433.500415:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776433.500418:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776433.500420:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776433.500420:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500422:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776433.500423:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776433.500425:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776433.500430:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776433.500433:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776433.500440:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099750528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776433.500446:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776433.500447:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776433.500451:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099750528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776433.500458:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776433.500461:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776433.500464:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776433.500464:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776433.500466:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776433.500469:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776433.500473:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18148 00000400:00000200:0.0:1775776433.500477:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776433.500479:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776433.500483:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776433.500489:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776433.500490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776433.500492:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500496:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776433.500496:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1775776433.500500:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776433.500504:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776433.500506:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776433.500509:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776433.500511:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776433.500516:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776433.500518:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776433.500523:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500528:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500532:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500536:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776433.500539:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776433.500543:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500547:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776433.500550:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776433.500553:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776433.500556:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500561:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776433.500564:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776433.500568:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500571:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776433.500575:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776433.500579:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776433.500583:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776433.500585:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500589:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776433.500591:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776433.500594:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776433.500600:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776433.500604:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776433.500610:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099750656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776433.500623:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776433.500626:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776433.500632:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099750656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776433.500643:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776433.500646:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776433.500652:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776433.500654:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776433.500656:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776433.500659:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776433.500663:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18148 00000400:00000200:0.0:1775776433.500671:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776433.500676:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776433.500680:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776433.500685:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500690:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500693:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500696:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776433.500698:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776433.500701:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776433.500704:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776433.500706:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776433.500710:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776433.500715:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776433.500720:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500723:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776433.500728:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500732:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776433.500737:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776433.500743:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776433.500747:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776433.500749:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776433.500752:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776433.500756:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500760:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776433.500763:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776433.500766:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500771:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776433.500775:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776433.500802:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776433.500806:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776433.500807:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776433.500809:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776433.500812:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776433.500814:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776433.500821:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776433.500826:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776433.500833:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099750784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776433.500858:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776433.500861:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776433.500867:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099750784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776433.500880:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776433.500886:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18158) 00000800:00000200:2.1:1775776433.501103:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776433.501118:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776433.501394:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776433.501396:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776433.501656:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776433.501658:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776433.501936:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776433.501939:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1775776434.267995:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776434.267998:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776434.268002:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776434.779888:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312822517, at 4312822528 00000100:00000040:0.0:1775776434.779901:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312822517, at 4312822528 00000100:00000010:1.0:1775776434.779904:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000010:0.0:1775776434.779908:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000020:00000001:1.0:1775776435.292062:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776435.292065:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776435.292068:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776436.316069:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776436.316073:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776436.316077:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776437.340040:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776437.340043:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776437.340047:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776438.364027:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776438.364030:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776438.364033:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776438.619962:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776438.619974:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776438.619977:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776438.619983:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776438.619987:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776438.619996:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18153 00000400:00000200:0.0:1775776438.620007:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776438.620014:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776438.620018:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776438.620022:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620025:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620028:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620031:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776438.620035:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776438.620037:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776438.620040:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776438.620043:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776438.620046:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776438.620048:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776438.620052:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620054:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620056:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620058:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776438.620061:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776438.620064:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620067:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776438.620070:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776438.620075:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776438.620078:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620081:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776438.620084:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776438.620096:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776438.620101:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776438.620106:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776438.620108:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620111:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776438.620114:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776438.620119:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776438.620127:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776438.620134:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776438.620145:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099750912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776438.620156:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776438.620161:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776438.620167:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099750912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776438.620193:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776438.620197:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776438.620200:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776438.620201:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776438.620203:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776438.620206:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776438.620208:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18153 00000400:00000200:0.0:1775776438.620212:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776438.620215:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776438.620218:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776438.620221:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620223:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620226:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620228:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776438.620231:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776438.620232:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776438.620234:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776438.620236:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776438.620240:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776438.620244:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776438.620249:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620253:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620258:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620263:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776438.620268:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776438.620273:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620277:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776438.620282:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776438.620289:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776438.620294:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620296:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776438.620301:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776438.620305:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620308:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776438.620311:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776438.620313:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776438.620317:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776438.620319:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620320:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776438.620321:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776438.620324:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776438.620330:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776438.620334:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776438.620339:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099751040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776438.620345:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776438.620348:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776438.620353:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099751040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776438.620369:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776438.620371:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776438.620377:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776438.620379:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776438.620381:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776438.620383:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776438.620387:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18153 00000400:00000200:0.0:1775776438.620391:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776438.620394:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776438.620396:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776438.620398:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620403:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620406:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620409:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776438.620413:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776438.620414:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776438.620415:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776438.620416:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776438.620420:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776438.620422:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776438.620427:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620428:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620433:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620435:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776438.620439:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776438.620444:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620447:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776438.620450:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776438.620452:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776438.620454:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620455:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776438.620456:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776438.620460:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620463:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776438.620468:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776438.620471:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776438.620475:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776438.620477:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620478:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776438.620479:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776438.620481:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776438.620485:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776438.620487:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776438.620492:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099751168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776438.620500:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776438.620503:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776438.620507:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099751168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776438.620515:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776438.620516:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776438.620519:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776438.620521:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:2.1:1775776438.620521:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000100:00000001:0.0:1775776438.620523:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776438.620525:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776438.620527:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776438.620531:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18153 00000400:00000200:0.0:1775776438.620535:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776438.620539:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776438.620541:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776438.620543:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620544:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620545:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620546:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776438.620549:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776438.620550:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776438.620552:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776438.620554:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776438.620557:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776438.620558:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776438.620561:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620564:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620565:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620568:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776438.620569:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776438.620571:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620573:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776438.620576:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776438.620577:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776438.620581:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620584:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776438.620588:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776438.620590:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620591:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776438.620594:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776438.620598:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776438.620602:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776438.620605:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620606:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776438.620609:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776438.620611:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776438.620614:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776438.620616:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776438.620621:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099751296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776438.620630:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776438.620633:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776438.620636:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099751296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776438.620641:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776438.620643:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776438.620645:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776438.620646:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776438.620648:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776438.620652:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776438.620654:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18153 00000400:00000200:0.0:1775776438.620659:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776438.620663:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776438.620665:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776438.620669:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620673:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620676:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620679:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776438.620683:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776438.620685:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776438.620687:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776438.620689:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776438.620691:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776438.620693:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776438.620697:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620700:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776438.620703:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620706:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776438.620709:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776438.620713:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776438.620717:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776438.620719:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776438.620722:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776438.620724:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620727:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776438.620729:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776438.620732:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620735:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776438.620740:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776438.620743:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776438.620745:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776438.620747:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776438.620749:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776438.620751:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776438.620753:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776438.620758:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776438.620762:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776438.620768:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099751424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776438.620797:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776438.620798:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776438.620802:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099751424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776438.620810:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776438.620816:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18164) 00000800:00000200:2.1:1775776438.621010:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776438.621012:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776438.621193:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776438.621195:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776438.621381:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776438.621383:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776438.621582:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776438.621584:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000020:00000001:1.0:1775776439.388006:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776439.388010:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776439.388014:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776439.899857:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312827637, at 4312827648 00000100:00000010:1.0:1775776439.899887:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000040:0.0:1775776439.899888:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312827637, at 4312827648 00000100:00000010:0.0:1775776439.899898:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776440.411968:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776440.411972:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776440.411976:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776441.436005:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776441.436017:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776441.436021:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776442.459998:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776442.460002:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776442.460006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776443.483987:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776443.483991:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776443.483994:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776443.739869:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776443.739876:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776443.739879:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776443.739882:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776443.739885:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776443.739890:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18159 00000400:00000200:0.0:1775776443.739898:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776443.739904:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776443.739907:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776443.739910:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.739913:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.739915:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.739918:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776443.739921:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776443.739923:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776443.739925:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776443.739928:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776443.739930:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776443.739933:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776443.739935:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.739938:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.739940:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.739941:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776443.739944:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776443.739946:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776443.739948:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776443.739950:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776443.739953:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776443.739954:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.739956:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776443.739957:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776443.739965:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776443.739968:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776443.739971:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776443.739972:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.739974:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776443.739976:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776443.739979:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776443.739986:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776443.739991:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776443.739997:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099751552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776443.740004:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776443.740007:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776443.740010:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099751552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776443.740031:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776443.740033:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776443.740035:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776443.740036:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776443.740037:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776443.740039:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776443.740041:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18159 00000400:00000200:0.0:1775776443.740043:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776443.740045:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776443.740047:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776443.740049:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740051:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.740053:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740054:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776443.740056:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776443.740057:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776443.740058:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776443.740060:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776443.740061:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776443.740062:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776443.740064:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740065:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.740068:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740070:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776443.740073:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776443.740076:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740079:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776443.740080:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776443.740084:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776443.740088:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740097:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776443.740101:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776443.740108:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740112:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776443.740115:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776443.740119:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776443.740122:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776443.740124:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740126:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776443.740128:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776443.740129:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776443.740132:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776443.740138:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776443.740143:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099751680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776443.740152:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776443.740154:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776443.740160:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099751680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776443.740176:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776443.740180:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776443.740187:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776443.740191:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776443.740194:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776443.740198:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776443.740205:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18159 00000400:00000200:0.0:1775776443.740209:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776443.740215:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776443.740217:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776443.740221:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740225:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.740227:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740229:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776443.740232:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776443.740234:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776443.740237:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776443.740238:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776443.740242:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776443.740245:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776443.740252:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740257:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.740261:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740263:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776443.740269:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776443.740276:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740281:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776443.740285:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776443.740288:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776443.740292:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740294:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776443.740296:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776443.740299:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740302:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776443.740308:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000800:00000200:2.1:1775776443.740310:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:0.0:1775776443.740318:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776443.740320:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776443.740321:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740322:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776443.740323:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776443.740324:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776443.740327:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776443.740332:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776443.740339:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099751808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776443.740347:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776443.740350:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776443.740353:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099751808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776443.740360:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776443.740363:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776443.740367:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776443.740370:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776443.740372:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776443.740376:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776443.740380:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18159 00000400:00000200:0.0:1775776443.740388:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776443.740392:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776443.740394:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776443.740396:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740397:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.740400:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740404:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776443.740406:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776443.740408:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776443.740411:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776443.740413:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776443.740415:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776443.740418:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776443.740421:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740425:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.740427:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740431:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776443.740434:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776443.740437:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740441:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776443.740444:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776443.740446:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776443.740449:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740452:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776443.740454:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776443.740458:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740460:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776443.740465:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776443.740468:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776443.740472:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776443.740474:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740476:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776443.740479:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776443.740480:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776443.740485:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776443.740488:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776443.740493:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099751936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776443.740504:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776443.740507:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776443.740512:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099751936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776443.740520:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776443.740523:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776443.740527:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776443.740529:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776443.740530:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776443.740532:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776443.740535:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18159 00000400:00000200:0.0:1775776443.740541:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776443.740544:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776443.740548:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776443.740552:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740556:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.740559:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740564:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776443.740568:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776443.740570:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776443.740573:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776443.740576:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776443.740580:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776443.740584:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776443.740590:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740593:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776443.740596:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740600:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776443.740603:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776443.740608:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776443.740611:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776443.740615:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776443.740619:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776443.740627:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740631:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776443.740635:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776443.740638:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740640:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776443.740646:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776443.740650:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776443.740656:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776443.740658:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776443.740662:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776443.740664:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776443.740668:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776443.740673:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776443.740677:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776443.740684:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099752064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775776443.740691:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776443.740699:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776443.740702:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776443.740710:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099752064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776443.740727:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776443.740734:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18169) 00000800:00000200:2.1:1775776443.740891:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776443.741032:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776443.741147:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775776444.507961:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776444.507964:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776444.507966:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776445.019889:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312832757, at 4312832768 00000100:00000040:1.0:1775776445.019889:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312832757, at 4312832768 00000100:00000010:0.0:1775776445.019903:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000010:1.0:1775776445.019905:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775776445.531951:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776445.531954:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776445.531956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776446.555936:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776446.555939:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776446.555942:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775776446.812016:0:301877:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775776446.812053:0:301877:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775776446.812063:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776446.812066:0:301877:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775776446.812071:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776446.812075:0:301877:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775776446.812080:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776446.812083:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7700. 00000400:00000200:1.0:1775776446.812089:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776446.812094:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776446.812100:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:1.0:1775776446.812102:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:1.0:1775776446.812108:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776446.812116:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454ded80 x1862033099751552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812151:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776446.812155:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099751552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812162:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.812164:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776446.812173:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776446.812176:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7000. 00000400:00000200:1.0:1775776446.812179:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776446.812182:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776446.812185:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720ddd0 00000400:00000010:1.0:1775776446.812187:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720ddd0. 00000100:00000001:1.0:1775776446.812190:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776446.812194:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dd880 x1862033099751680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812203:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776446.812206:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099751680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812213:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.812217:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776446.812225:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776446.812230:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:1.0:1775776446.812233:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776446.812240:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776446.812245:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:1.0:1775776446.812249:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:1.0:1775776446.812253:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776446.812258:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454df480 x1862033099751808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812271:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776446.812275:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df480 x1862033099751808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812287:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.812290:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776446.812299:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776446.812304:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7c00. 00000400:00000200:1.0:1775776446.812310:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776446.812317:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776446.812322:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d550 00000400:00000010:1.0:1775776446.812325:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d550. 00000100:00000001:1.0:1775776446.812328:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776446.812334:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dea00 x1862033099751936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812369:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776446.812376:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dea00 x1862033099751936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812403:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.812407:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776446.812418:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776446.812423:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:1.0:1775776446.812427:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776446.812432:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776446.812437:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d198 00000400:00000010:1.0:1775776446.812441:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d198. 00000100:00000001:1.0:1775776446.812445:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776446.812453:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dd500 x1862033099752064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812472:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776446.812479:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd500 x1862033099752064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.812490:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.812494:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775776447.579925:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776447.579928:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776447.579930:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776448.603993:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776448.603997:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776448.603999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776448.859916:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776448.859924:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776448.859926:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776448.859929:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776448.859932:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776448.859937:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18164 00000400:00000200:1.0:1775776448.859945:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776448.859949:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776448.859952:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776448.859954:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.859956:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.859958:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.859959:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776448.859963:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776448.859964:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776448.859966:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776448.859967:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776448.859968:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776448.859970:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776448.859971:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.859972:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.859973:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.859975:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776448.859977:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776448.859979:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776448.859980:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776448.859982:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776448.859984:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776448.859985:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.859987:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776448.859988:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776448.859993:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776448.859995:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776448.859998:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776448.859999:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860001:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776448.860004:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776448.860009:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776448.860015:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776448.860021:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776448.860030:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099752192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776448.860039:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776448.860043:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776448.860047:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099752192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776448.860075:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776448.860078:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776448.860081:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776448.860082:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776448.860084:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776448.860086:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776448.860088:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18164 00000400:00000200:1.0:1775776448.860092:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776448.860095:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776448.860097:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776448.860099:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860101:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.860103:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860105:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776448.860107:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776448.860108:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776448.860110:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776448.860112:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776448.860114:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776448.860116:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776448.860119:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860121:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.860124:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860127:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776448.860130:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776448.860135:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860139:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776448.860142:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776448.860149:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776448.860155:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860159:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776448.860165:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776448.860169:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860173:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776448.860184:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776448.860188:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776448.860194:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776448.860197:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860200:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776448.860201:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776448.860203:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776448.860207:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776448.860211:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776448.860218:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099752320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776448.860231:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776448.860235:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776448.860240:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099752320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776448.860251:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776448.860256:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776448.860262:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776448.860265:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776448.860267:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776448.860273:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776448.860278:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18164 00000400:00000200:1.0:1775776448.860285:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776448.860292:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776448.860296:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776448.860302:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860305:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.860310:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860314:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776448.860317:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776448.860318:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776448.860320:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776448.860324:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776448.860327:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776448.860331:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776448.860337:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860340:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.860342:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860346:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776448.860350:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776448.860355:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860361:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776448.860364:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776448.860369:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776448.860373:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860376:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776448.860379:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776448.860381:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860385:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776448.860390:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776448.860394:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776448.860398:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776448.860400:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860403:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776448.860405:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776448.860406:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776448.860411:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776448.860417:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776448.860424:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099752448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776448.860432:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776448.860435:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776448.860439:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776448.860442:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000040:1.0:1775776448.860450:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099752448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776448.860462:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776448.860467:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776448.860472:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776448.860473:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776448.860475:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776448.860478:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776448.860481:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18164 00000400:00000200:1.0:1775776448.860484:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776448.860489:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776448.860491:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776448.860494:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860498:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.860501:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860504:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776448.860507:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776448.860509:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776448.860511:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776448.860513:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776448.860515:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776448.860517:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776448.860520:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860522:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.860523:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860525:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776448.860530:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776448.860536:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860541:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776448.860543:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776448.860546:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776448.860550:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860552:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776448.860554:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776448.860556:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860558:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776448.860561:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776448.860563:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776448.860565:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776448.860566:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860568:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776448.860569:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776448.860570:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776448.860574:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776448.860576:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776448.860580:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099752576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776448.860586:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776448.860588:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776448.860592:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099752576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776448.860599:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776448.860601:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776448.860604:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776448.860606:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776448.860607:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776448.860610:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776448.860612:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18164 00000400:00000200:1.0:1775776448.860615:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776448.860618:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776448.860620:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776448.860622:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860624:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.860625:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860627:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776448.860628:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776448.860629:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776448.860631:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776448.860632:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776448.860634:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776448.860635:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776448.860638:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860639:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776448.860641:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860642:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776448.860644:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776448.860646:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776448.860648:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776448.860651:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776448.860653:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776448.860655:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860657:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776448.860659:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776448.860661:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860662:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776448.860665:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776448.860667:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776448.860676:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776448.860677:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776448.860679:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776448.860680:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776448.860681:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776448.860684:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776448.860687:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776448.860690:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099752704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776448.860696:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776448.860698:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776448.860733:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099752704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776448.860741:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776448.860746:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18174) 00000800:00000200:2.1:1775776448.863045:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776448.863057:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776448.863243:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776448.863244:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776448.863424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776448.863426:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776448.863617:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776448.863619:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1775776449.627971:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776449.627973:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776449.627975:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776450.139838:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312837879, at 4312837888 00000100:00000040:1.0:1775776450.139838:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312837879, at 4312837888 00000100:00000010:0.0:1775776450.139852:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000010:1.0:1775776450.139854:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000020:00000001:1.0:1775776450.651926:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776450.651930:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776450.651933:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776451.675957:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776451.675961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776451.675965:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776452.700058:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776452.700062:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776452.700065:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776453.724011:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776453.724016:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776453.724021:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776453.979920:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776453.979933:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776453.979936:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776453.979940:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776453.979945:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776453.979952:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18169 00000400:00000200:1.0:1775776453.979963:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776453.979970:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776453.979973:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776453.979976:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.979979:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.979981:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.979983:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776453.979986:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776453.979988:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776453.979991:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776453.979993:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776453.979995:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776453.979996:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776453.979998:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.979999:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.980001:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980002:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776453.980005:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776453.980009:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980011:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776453.980013:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776453.980016:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776453.980018:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980020:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776453.980022:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776453.980032:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776453.980035:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776453.980039:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776453.980040:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980043:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776453.980046:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776453.980051:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776453.980062:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776453.980069:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776453.980078:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099752832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776453.980090:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776453.980096:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776453.980102:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099752832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776453.980135:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776453.980139:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776453.980143:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776453.980144:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776453.980146:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776453.980149:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776453.980151:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18169 00000400:00000200:1.0:1775776453.980155:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776453.980157:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776453.980159:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776453.980162:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980163:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.980165:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980167:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776453.980169:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776453.980170:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776453.980171:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776453.980172:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776453.980174:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776453.980176:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776453.980178:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980179:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.980181:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980183:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776453.980185:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776453.980187:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980188:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776453.980190:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776453.980203:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776453.980205:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980206:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776453.980214:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776453.980216:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980222:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776453.980226:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776453.980230:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776453.980234:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776453.980235:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980237:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776453.980238:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776453.980240:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776453.980243:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776453.980249:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776453.980253:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099752960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776453.980275:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776453.980281:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776453.980294:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099752960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776453.980321:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776453.980326:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776453.980337:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776453.980341:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776453.980343:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776453.980352:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776453.980360:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18169 00000400:00000200:1.0:1775776453.980368:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776453.980372:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776453.980374:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776453.980382:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980388:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.980392:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980394:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776453.980397:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776453.980399:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776453.980408:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776453.980412:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776453.980416:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776453.980424:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776453.980432:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980434:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.980440:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980444:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776453.980448:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776453.980459:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980463:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776453.980469:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776453.980476:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776453.980483:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980490:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776453.980494:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776453.980505:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980511:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776453.980521:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776453.980527:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776453.980534:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776453.980537:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980540:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776453.980543:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776453.980549:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776453.980554:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776453.980559:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776453.980565:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099753088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776453.980579:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776453.980589:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776453.980598:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099753088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776453.980651:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776453.980657:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776453.980664:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776453.980670:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776453.980675:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776453.980685:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776453.980692:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18169 00000400:00000200:1.0:1775776453.980706:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776453.980715:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776453.980720:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776453.980725:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980728:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.980730:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980735:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776453.980744:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776453.980749:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776453.980754:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776453.980760:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776453.980766:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776453.980767:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776453.980771:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980775:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.980776:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980820:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776453.980822:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776453.980826:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776453.980835:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776453.980838:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776453.980844:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776453.980848:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980857:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776453.980861:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776453.980865:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980870:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776453.980875:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776453.980878:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776453.980881:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776453.980883:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.980888:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776453.980890:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776453.980892:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776453.980898:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000010:1.0:1775776453.980899:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776453.980907:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776453.980907:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00100000:1.0:1775776453.980916:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099753216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776453.980938:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776453.980943:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776453.980949:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099753216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776453.980962:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776453.980967:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776453.980975:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776453.980977:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776453.980978:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776453.980981:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776453.980987:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18169 00000400:00000200:1.0:1775776453.980995:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776453.981000:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776453.981002:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776453.981007:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.981013:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.981019:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.981027:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776453.981035:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776453.981040:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776453.981047:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776453.981050:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776453.981058:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776453.981062:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776453.981065:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.981071:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776453.981075:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776453.981078:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776453.981083:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776453.981092:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776453.981105:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776453.981108:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776453.981115:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776453.981121:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.981125:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776453.981132:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776453.981142:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.981145:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776453.981148:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776453.981154:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776453.981157:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776453.981158:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776453.981161:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776453.981165:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776453.981167:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776453.981172:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776453.981176:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776453.981185:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099753344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776453.981202:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776453.981205:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776453.981217:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099753344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776453.981238:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776453.981246:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18179) 00000800:00000200:2.1:1775776453.981546:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776453.981549:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776453.981890:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776453.981893:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776453.982171:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776453.982174:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776453.982461:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776453.982463:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775776454.747956:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776454.747960:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776454.747963:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776455.259877:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312842998, at 4312843008 00000100:00000040:1.0:1775776455.259891:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312842998, at 4312843008 00000100:00000010:0.0:1775776455.259897:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000010:1.0:1775776455.259900:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000020:00000001:1.0:1775776455.772028:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776455.772032:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776455.772036:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776456.796026:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776456.796030:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776456.796033:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776457.820015:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776457.820020:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776457.820025:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776458.843975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776458.843979:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776458.843981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776459.099938:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776459.099949:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776459.099953:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776459.099958:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776459.099963:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776459.099972:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18174 00000400:00000200:0.0:1775776459.099985:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776459.099992:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776459.099996:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776459.100000:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100004:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100007:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100010:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776459.100015:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776459.100017:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776459.100021:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776459.100025:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776459.100027:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776459.100030:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776459.100034:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100037:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100039:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100041:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776459.100046:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776459.100049:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100053:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776459.100056:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776459.100061:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776459.100064:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100067:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776459.100070:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776459.100083:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776459.100088:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776459.100093:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776459.100095:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100099:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776459.100103:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776459.100108:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776459.100119:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776459.100135:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776459.100147:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099753472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776459.100160:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776459.100166:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776459.100171:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099753472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776459.100200:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776459.100204:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776459.100207:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776459.100209:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776459.100211:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776459.100214:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776459.100217:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18174 00000400:00000200:0.0:1775776459.100221:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776459.100226:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776459.100229:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776459.100232:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100235:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100238:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100241:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776459.100245:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776459.100249:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776459.100253:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776459.100256:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776459.100260:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776459.100265:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776459.100276:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100281:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100286:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100291:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776459.100296:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776459.100305:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100309:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776459.100312:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776459.100319:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776459.100324:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100329:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776459.100335:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776459.100341:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100344:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776459.100353:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776459.100358:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776459.100361:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776459.100365:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100367:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776459.100370:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776459.100375:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776459.100384:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776459.100391:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776459.100402:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099753600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776459.100419:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776459.100423:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776459.100430:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099753600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776459.100448:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776459.100454:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776459.100463:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776459.100469:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776459.100473:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776459.100480:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776459.100484:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18174 00000400:00000200:0.0:1775776459.100500:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776459.100508:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776459.100511:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776459.100514:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100516:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100521:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100539:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776459.100543:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776459.100546:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776459.100547:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776459.100551:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000800:00000200:2.1:1775776459.100551:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:0.0:1775776459.100555:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000800:00000200:2.1:1775776459.100557:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:0.0:1775776459.100558:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776459.100565:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100569:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100571:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100574:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776459.100576:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776459.100578:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100579:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776459.100585:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776459.100588:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776459.100592:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100595:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776459.100597:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776459.100601:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100604:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776459.100609:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776459.100613:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776459.100618:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776459.100621:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100626:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776459.100629:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776459.100631:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776459.100637:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776459.100644:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776459.100655:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099753728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776459.100671:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776459.100676:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776459.100683:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099753728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776459.100698:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776459.100700:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776459.100705:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776459.100707:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776459.100711:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776459.100714:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776459.100719:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18174 00000400:00000200:0.0:1775776459.100726:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776459.100730:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776459.100732:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776459.100737:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100742:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100746:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100752:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776459.100754:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776459.100755:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776459.100758:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776459.100760:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776459.100761:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776459.100764:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776459.100768:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100771:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100775:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100806:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776459.100810:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776459.100816:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100820:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776459.100822:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776459.100824:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776459.100826:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100829:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776459.100833:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776459.100836:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100839:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776459.100842:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776459.100846:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776459.100850:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776459.100853:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.100857:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776459.100859:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776459.100861:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776459.100866:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776459.100873:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776459.100880:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099753856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776459.100888:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776459.100892:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776459.100899:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099753856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776459.100909:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776459.100912:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776459.100918:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776459.100922:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776459.100925:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776459.100933:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776459.100939:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18174 00000400:00000200:0.0:1775776459.100951:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776459.100960:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776459.100965:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776459.100975:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100980:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.100986:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.100992:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776459.100998:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776459.101002:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776459.101007:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776459.101011:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776459.101017:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776459.101022:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776459.101030:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.101035:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776459.101040:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776459.101044:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776459.101050:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776459.101055:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776459.101058:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776459.101063:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776459.101068:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776459.101074:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.101079:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776459.101083:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776459.101089:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.101093:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776459.101101:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776459.101106:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776459.101108:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776459.101110:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776459.101112:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776459.101116:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776459.101120:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776459.101128:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776459.101134:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776459.101146:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099753984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776459.101189:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776459.101194:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776459.101203:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099753984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776459.101230:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776459.101238:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18184) 00000800:00000200:2.1:1775776459.101265:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776459.101267:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776459.101459:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776459.101460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776459.101663:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776459.101680:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776459.101987:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776459.101989:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1775776459.867991:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776459.867995:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776459.867998:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776460.379927:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312848117, at 4312848128 00000100:00000040:1.0:1775776460.379927:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312848117, at 4312848128 00000100:00000010:1.0:1775776460.379946:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000010:0.0:1775776460.379954:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000020:00000001:1.0:1775776460.892007:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776460.892011:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776460.892014:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776461.915999:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776461.916003:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776461.916006:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776462.940017:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776462.940021:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776462.940026:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776463.964010:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776463.964024:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776463.964028:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776464.219929:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776464.219941:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776464.219944:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776464.219951:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776464.219956:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776464.219966:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18179 00000400:00000200:0.0:1775776464.219980:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776464.219988:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776464.219993:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776464.219997:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220001:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.220004:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220007:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776464.220012:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776464.220015:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776464.220018:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776464.220021:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776464.220024:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776464.220027:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776464.220030:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220033:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.220035:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220037:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776464.220040:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776464.220043:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220046:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776464.220050:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776464.220054:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776464.220057:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220060:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776464.220062:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776464.220074:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776464.220078:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776464.220082:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776464.220084:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220088:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776464.220091:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776464.220096:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776464.220107:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776464.220114:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776464.220124:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099754112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776464.220137:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776464.220142:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776464.220148:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099754112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776464.220174:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776464.220177:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776464.220180:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776464.220189:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776464.220191:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776464.220195:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776464.220199:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18179 00000400:00000200:0.0:1775776464.220204:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776464.220208:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776464.220211:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776464.220218:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220221:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.220227:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220231:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776464.220235:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776464.220238:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776464.220242:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776464.220246:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776464.220251:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776464.220255:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776464.220262:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220265:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.220270:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220274:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776464.220279:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776464.220285:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220291:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776464.220304:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776464.220313:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776464.220318:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220322:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776464.220328:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776464.220335:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220338:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776464.220344:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776464.220348:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776464.220354:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776464.220356:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220358:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776464.220359:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776464.220361:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776464.220369:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776464.220377:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776464.220387:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099754240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776464.220406:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776464.220410:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776464.220418:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099754240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776464.220448:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776464.220453:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776464.220460:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776464.220464:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776464.220466:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776464.220471:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776464.220478:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18179 00000400:00000200:0.0:1775776464.220487:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776464.220495:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776464.220502:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776464.220510:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220515:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.220521:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220525:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776464.220530:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776464.220531:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776464.220534:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776464.220536:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776464.220542:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776464.220549:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776464.220556:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220561:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.220563:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220568:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776464.220571:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776464.220577:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220584:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776464.220587:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776464.220593:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775776464.220595:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776464.220597:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220601:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.1:1775776464.220601:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000008:00100000:0.0:1775776464.220605:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776464.220610:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220615:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776464.220621:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776464.220625:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776464.220628:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776464.220631:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220635:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776464.220639:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776464.220643:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776464.220648:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776464.220655:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776464.220664:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099754368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776464.220684:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776464.220688:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776464.220698:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099754368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776464.220719:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776464.220724:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776464.220741:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776464.220746:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776464.220748:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776464.220753:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776464.220758:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18179 00000400:00000200:0.0:1775776464.220764:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776464.220769:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776464.220774:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776464.220803:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220807:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.220810:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220813:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776464.220817:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776464.220819:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776464.220823:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776464.220826:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776464.220830:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776464.220836:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776464.220844:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220850:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.220852:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220856:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776464.220861:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776464.220866:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776464.220869:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776464.220874:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776464.220880:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776464.220885:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220891:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776464.220897:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776464.220903:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220908:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776464.220914:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776464.220920:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776464.220927:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776464.220931:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.220937:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776464.220940:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776464.220944:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776464.220953:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776464.220960:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776464.220970:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099754496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776464.220992:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776464.220997:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776464.221007:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099754496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776464.221022:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776464.221028:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776464.221035:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776464.221039:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776464.221042:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776464.221049:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776464.221057:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18179 00000400:00000200:0.0:1775776464.221067:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776464.221074:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776464.221080:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776464.221083:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.221087:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.221094:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.221100:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776464.221106:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776464.221109:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776464.221114:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776464.221117:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776464.221124:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776464.221131:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776464.221139:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.221144:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776464.221149:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776464.221155:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776464.221159:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776464.221165:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776464.221171:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776464.221177:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776464.221182:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776464.221189:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.221194:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776464.221198:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776464.221204:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.221208:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776464.221217:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776464.221223:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776464.221229:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776464.221232:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776464.221238:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776464.221241:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776464.221245:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776464.221253:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776464.221260:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776464.221270:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099754624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775776464.221286:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776464.221293:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000001:0.0:1775776464.221302:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776464.221306:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776464.221314:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099754624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776464.221336:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776464.221343:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18189) 00000800:00000200:2.1:1775776464.221662:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776464.221665:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776464.221988:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776464.221990:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776464.222388:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776464.222391:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775776464.988035:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776464.988040:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776464.988044:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776465.499905:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312853237, at 4312853248 00000100:00000040:1.0:1775776465.499905:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312853237, at 4312853248 00000100:00000010:0.0:1775776465.499927:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000010:1.0:1775776465.499933:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000020:00000001:1.0:1775776466.011967:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776466.011972:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776466.011976:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776467.035955:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776467.035959:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776467.035962:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1775776468.060065:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776468.060070:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776468.060075:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776469.084011:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776469.084015:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776469.084018:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776469.340569:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776469.340574:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776469.341377:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776469.341380:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776469.341839:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776469.341842:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776469.342286:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776469.342289:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776469.342616:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776469.342618:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1775776470.108026:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776470.108030:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776470.108033:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776470.619918:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312858357, at 4312858368 00000100:00000040:1.0:1775776470.619918:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312858357, at 4312858368 00000100:00000010:1.0:1775776470.619938:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344a80. 00000100:00000010:0.0:1775776470.619941:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000020:00000001:0.0:1775776471.131994:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776471.131999:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776471.132004:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776472.155987:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776472.155999:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776472.156002:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1775776473.179936:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776473.179939:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776473.179941:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1775776474.203933:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776474.203936:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776474.203938:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775776474.460299:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776474.460304:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776474.460808:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776474.460810:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776474.461015:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776474.461017:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776474.461168:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776474.461169:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000400:00000200:3.0:1775776474.461340:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776474.461342:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4136:4136:256:4294967295] 192.168.201.139@tcp LPNI seq info [4136:4136:8:4294967295] 00000400:00000200:3.0:1775776474.461344:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776474.461348:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776474.461351:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776474.461353:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776474.461355:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776474.461356:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775776474.461359:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776474.461360:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:3.0:1775776474.461362:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776474.461363:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776474.461365:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776474.461367:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776474.461368:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776474.461369:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776474.461371:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776474.461372:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776499 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461377:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776474.461378:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776499 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461382:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776474.461383:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776474.461384:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776474.461386:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776474.461387:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776474.461388:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312862214 00000100:00000001:3.0:1775776474.461390:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312863477 00000100:00000001:3.0:1775776474.461391:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312862214 00000100:00000001:3.0:1775776474.461393:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312863477 00000100:00000001:3.0:1775776474.461394:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776474.461395:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776474.461396:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776474.461398:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776474/real 1775776474] req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776474.461402:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776474.461403:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776474.461404:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776474.461405:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776474.461405:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776474.461406:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776474.461408:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776474.461413:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461416:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776474.461417:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461417:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776474.461418:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776474.461420:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461423:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776474.461425:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461428:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776474.461430:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461431:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776474.461431:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461433:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776474.461434:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776474.461435:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776474.461437:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776474.461438:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776474.461440:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776474.461444:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099755904:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776474.461446:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461447:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776474.461449:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099755520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461452:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776474.461453:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776474.461454:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776474.461455:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776474.461456:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776474.461457:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776474.461458:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776474.461459:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507400. 02000000:00000001:3.0:1775776474.461460:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776474.461461:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776474.461462:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:3.0:1775776474.461463:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776474.461464:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461465:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776474.461466:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f800 x1862033099755648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461470:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776474.461471:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776474.461472:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776474.461473:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776474.461473:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776474.461474:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776474.461475:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776474.461476:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506800. 02000000:00000001:3.0:1775776474.461477:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776474.461478:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776474.461479:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f800. 00000100:00000001:3.0:1775776474.461480:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776474.461481:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461482:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776474.461483:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ed80 x1862033099755776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461486:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776474.461487:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776474.461488:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776474.461488:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776474.461489:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776474.461490:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776474.461490:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776474.461491:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505400. 02000000:00000001:3.0:1775776474.461492:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776474.461493:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776474.461493:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ed80. 00000100:00000001:3.0:1775776474.461495:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776474.461495:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461496:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776474.461497:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c700 x1862033099755904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776499 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776474.461500:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776474.461501:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776474.461502:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776474.461503:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776474.461503:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776474.461504:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776474.461505:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776474.461505:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:3.0:1775776474.461506:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776474.461507:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776474.461507:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c700. 00000100:00000001:3.0:1775776474.461509:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776474.461510:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461511:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776474.461513:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776474.461514:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776474.461515:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776474.461516:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776475.163884:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776475.163891:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776475.163893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776475.163899:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776475.163903:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776475.163907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776475.163910:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776475.163913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776475.163915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776475.227934:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776475.227942:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776475.227945:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776475.227953:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776475.227958:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776475.227963:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776475.227966:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776475.227972:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776475.227980:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776475.228057:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776475.228062:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776475.228066:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776475.739892:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312863477, at 4312863488 00000100:00000040:0.0:1775776475.739897:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312863477, at 4312863488 00000100:00000010:0.0:1775776475.739909:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000100:00000010:1.0:1775776475.739912:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344180. 00000100:00000001:3.0:1775776476.187925:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776476.187932:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776476.187933:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776476.187939:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776476.187943:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776476.187948:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776476.187951:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776476.187954:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776476.187957:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776476.251941:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776476.251948:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776476.251950:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776476.251955:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776476.251962:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776476.251967:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776476.251971:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776476.251974:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776476.251979:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776476.251998:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776476.252002:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776476.252006:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776477.211918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776477.211926:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776477.211928:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776477.211934:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776477.211939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776477.211943:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776477.211946:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776477.211948:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776477.211952:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776477.275879:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776477.275890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776477.275892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776477.275898:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776477.275904:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776477.275910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776477.275915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776477.275919:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776477.275926:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776477.275992:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776477.275996:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776477.275999:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776478.235884:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776478.235890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776478.235891:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776478.235895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776478.235898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776478.235901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776478.235903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776478.235905:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776478.235907:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776478.299868:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776478.299877:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776478.299881:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776478.299888:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776478.299894:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776478.299900:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776478.299906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776478.299910:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776478.299914:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776478.299969:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776478.299973:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776478.299977:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776479.259879:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776479.259885:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776479.259887:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776479.259891:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776479.259894:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776479.259897:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.259899:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776479.259901:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.259903:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.323895:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776479.323904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776479.323907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776479.323914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776479.323920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776479.323925:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.323930:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776479.323935:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.323942:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776479.324032:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776479.324036:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776479.324039:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776479.579935:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776479.579947:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776479.579950:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776479.579954:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776479.579959:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776479.579967:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18194 00000400:00000200:1.0:1775776479.579976:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776479.579983:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776479.579986:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776479.579990:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.579993:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776479.579996:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.579999:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776479.580003:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776479.580005:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776479.580008:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776479.580011:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776479.580021:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776479.580024:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776479.580028:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.580030:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776479.580033:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.580035:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776479.580039:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776479.580043:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776479.580046:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776479.580049:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776479.580052:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776479.580055:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.580058:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776479.580061:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776479.580071:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776479.580075:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776479.580079:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776479.580081:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.580084:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776479.580087:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776479.580091:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776479.580099:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776479.580106:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776479.580115:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776479.580126:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776479.580131:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776479.580136:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776479.580160:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776479.580164:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776479.580182:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776479.580183:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776479.580185:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776479.580189:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776479.580192:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18194 00000400:00000200:1.0:1775776479.580196:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776479.580200:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776479.580204:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776479.580208:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.580211:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776479.580213:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776479.580215:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776479.580216:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776479.580219:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775776479.580220:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775776479.580222:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1775776479.580223:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776479.580226:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776479.580230:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000100:00000040:3.0:1775776479.580230:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775776479.580235:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776479.580241:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776479.580248:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1775776479.580249:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776479.580250:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.580253:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1775776479.580255:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775776479.580258:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000001:3.0:1775776479.580259:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776479.580263:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:3.0:1775776479.580263:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775776479.580269:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00100000:3.0:1775776479.580270:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099756032:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776479.580274:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776479.580277:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776479.580278:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776479.580282:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1775776479.580285:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 02000000:00000010:3.0:1775776479.580287:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 00000100:00000001:1.0:1775776479.580289:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776479.580290:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776479.580293:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1775776479.580297:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:1.0:1775776479.580300:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1775776479.580301:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 02000000:00000001:1.0:1775776479.580306:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776479.580308:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000001:1.0:1775776479.580309:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776479.580315:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00000200:3.0:1775776479.580317:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099756032, portal 25 00000008:00000001:1.0:1775776479.580322:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.580322:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776479.580323:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601024 : -114841421950592 : ffff978d65d6d180) 00000100:00000001:1.0:1775776479.580327:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776479.580331:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775776479.580333:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776479.580339:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 00000100:00000001:3.0:1775776479.580339:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1775776479.580345:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:3.0:1775776479.580345:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776479.580347:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 02000000:00000001:1.0:1775776479.580348:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.580352:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:3.0:1775776479.580353:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099756032, offset 0 02000000:00000001:1.0:1775776479.580356:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776479.580359:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776479.580360:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776479.580369:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.580373:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 02000000:00000010:1.0:1775776479.580377:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000400:00000200:3.0:1775776479.580378:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775776479.580384:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776479.580384:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776479.580388:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776479.580394:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00100000:1.0:1775776479.580396:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775776479.580401:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4137:4137:256:4294967295] 192.168.201.139@tcp LPNI seq info [4137:4137:8:4294967295] 00000400:00000200:3.0:1775776479.580412:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:1.0:1775776479.580418:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776479.580422:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776479.580425:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000040:1.0:1775776479.580431:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1775776479.580434:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.580439:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776479.580448:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776479.580451:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775776479.580452:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00080000:1.0:1775776479.580456:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776479.580460:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776479.580462:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776479.580465:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.1:1775776479.580469:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776479.580471:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:3.1:1775776479.580472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00080000:1.0:1775776479.580477:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18194 00000800:00000100:3.0:1775776479.580479:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.580483:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:1.0:1775776479.580487:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.580488:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776479.580493:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000200:3.0:1775776479.580494:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1775776479.580499:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1775776479.580500:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776479.580502:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776479.580505:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776479.580509:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776479.580513:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.580519:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776479.580524:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776479.580525:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775776479.580528:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000040:3.0:1775776479.580530:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775776479.580533:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776479.580536:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776479.580539:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776479.580541:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775776479.580542:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000100:00000001:3.0:1775776479.580543:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1775776479.580547:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000001:3.0:1775776479.580548:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776479.580551:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:3.0:1775776479.580555:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:1.0:1775776479.580559:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776479.580559:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312868597 00000100:00000001:1.0:1775776479.580564:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:3.0:1775776479.580565:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344360. 00000100:00000040:1.0:1775776479.580568:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.580572:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776479.580572:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000020:00000040:1.0:1775776479.580576:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00000001:3.0:1775776479.580581:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312868597 00000100:00080000:1.0:1775776479.580582:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000010:3.0:1775776479.580585:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344c60. 00000100:00000001:1.0:1775776479.580587:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.580589:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1775776479.580591:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775776479.580592:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776479.580595:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776479.580596:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776479.580597:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1775776479.580601:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776479.580602:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776479/real 1775776479] req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776479.580605:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776479.580609:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000400:00000001:3.0:1775776479.580613:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:1.0:1775776479.580614:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1775776479.580615:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775776479.580619:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776479.580619:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776479.580624:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776479.580626:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000010:1.0:1775776479.580627:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000100:00000001:3.0:1775776479.580628:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1775776479.580632:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000100:00000200:3.0:1775776479.580632:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776479.580637:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776479.580638:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.580641:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776479.580652:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:3.0:1775776479.580654:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776479.580655:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776479.580663:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:3.0:1775776479.580663:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776479.580666:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776479.580669:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.580669:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.580672:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776479.580677:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776479.580681:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776479.580688:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776479.580696:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776479.580703:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776479.580708:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.580708:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776479.580713:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.580714:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1775776479.580717:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.580717:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.580719:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776479.580722:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776479.580727:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776479.580733:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776479.580737:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1775776479.580741:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776479.580744:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776479.580746:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776479.580757:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776479.580760:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776479.580763:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775776479.580766:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099756032:192.168.201.139@tcp:250:kworker.0 00000100:00000001:1.0:1775776479.580767:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776479.580774:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18194 00000100:00000001:3.0:1775776479.580775:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775776479.580808:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776479.580813:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:3.0:1775776479.580818:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1775776479.580819:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776479.580826:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000040:3.0:1775776479.580827:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d180 x1862033099756032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776479.580830:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776479.580835:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.580841:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776479.580844:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776479.580845:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776479.580847:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776479.580849:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776479.580854:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776479.580860:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776479.580866:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.580870:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776479.580871:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000040:1.0:1775776479.580874:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000001:3.0:1775776479.580876:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1775776479.580877:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000010:3.0:1775776479.580880:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 00000020:00000040:1.0:1775776479.580882:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 02000000:00000001:3.0:1775776479.580886:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1775776479.580888:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000020:00000001:3.0:1775776479.580890:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1775776479.580892:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776479.580894:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776479.580895:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1775776479.580899:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:1.0:1775776479.580900:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1775776479.580902:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39800. 02000000:00000001:1.0:1775776479.580903:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.580908:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:3.0:1775776479.580909:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000008:00100000:1.0:1775776479.580910:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 02000000:00000001:3.0:1775776479.580912:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000008:00000001:1.0:1775776479.580916:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776479.580916:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d180. 00000100:00000001:1.0:1775776479.580918:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776479.580921:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776479.580923:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776479.580925:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776479.580927:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.580930:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776479.580932:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775776479.580932:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1775776479.580935:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.580936:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1775776479.580940:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:3.0:1775776479.580940:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776479.580945:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.580948:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1775776479.580949:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776479.580952:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.580958:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776479.580959:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776479.580962:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.580962:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776479.580967:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776479.580968:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776479.580976:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776479.580977:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776479.580981:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776479.580989:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776479.580992:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776479.580995:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776479.581001:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776479.581001:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1775776479.581004:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18194 00000100:00000001:3.0:1775776479.581005:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776479.581009:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:3.0:1775776479.581009:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1775776479.581013:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 02000000:00000001:3.0:1775776479.581014:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1775776479.581015:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776479.581021:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000001:3.0:1775776479.581021:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.581025:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776479.581029:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00100000:3.0:1775776479.581030:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756160:192.168.201.139@tcp:38:kworker.0 00000100:00000001:1.0:1775776479.581032:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776479.581037:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775776479.581040:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1775776479.581041:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776479.581044:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1775776479.581045:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1775776479.581046:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776479.581049:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 02000000:00000001:3.0:1775776479.581049:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776479.581055:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 02000000:00000001:3.0:1775776479.581055:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1775776479.581059:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776479.581061:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776479.581062:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000010:3.0:1775776479.581062:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 00000100:00000001:1.0:1775776479.581068:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:3.0:1775776479.581068:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775776479.581074:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000400:00000200:3.0:1775776479.581078:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:1.0:1775776479.581079:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776479.581084:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776479.581085:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 02000000:00000001:1.0:1775776479.581087:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776479.581091:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1775776479.581092:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 02000000:00000001:1.0:1775776479.581093:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.581097:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776479.581100:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000200:3.0:1775776479.581100:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099756160, portal 10 00000008:00000001:1.0:1775776479.581102:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581105:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776479.581106:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581109:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287600128 : -114841421951488 : ffff978d65d6ce00) 00000100:00000010:1.0:1775776479.581111:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776479.581115:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776479.581117:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776479.581119:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776479.581122:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:3.0:1775776479.581122:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776479.581125:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776479.581127:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776479.581133:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:3.0:1775776479.581134:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775776479.581136:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776479.581137:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776479.581142:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00100000:1.0:1775776479.581144:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1775776479.581146:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099756160, offset 0 00000400:00000200:3.0:1775776479.581155:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776479.581163:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.581183:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:1.0:1775776479.581186:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1775776479.581187:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775776479.581190:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776479.581192:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000040:1.0:1775776479.581196:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775776479.581197:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776479.581202:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776479.581209:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4138:4138:256:4294967295] 192.168.201.139@tcp LPNI seq info [4138:4138:8:4294967295] 00000100:00000001:1.0:1775776479.581216:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776479.581217:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:1.0:1775776479.581222:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18205) 00000400:00000200:3.0:1775776479.581228:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776479.581232:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.581235:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776479.581239:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776479.581241:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776479.581246:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776479.581247:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776479.581250:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.581252:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776479.581254:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776479.581256:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776479.581258:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776479.581260:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776479.581262:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776479.581265:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581271:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776479.581273:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581277:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581279:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776479.581281:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776479.581283:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776479.581286:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776479.581287:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312867333 00000100:00000001:3.0:1775776479.581289:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312868597 00000100:00000001:3.0:1775776479.581291:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312867333 00000100:00000001:3.0:1775776479.581292:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312868597 00000100:00000001:3.0:1775776479.581293:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776479.581295:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581296:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776479.581298:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776479/real 1775776479] req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776479.581303:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776479.581304:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776479.581305:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776479.581307:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776479.581308:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776479.581309:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776479.581311:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776479.581317:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581325:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776479.581326:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581327:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.581328:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776479.581330:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581334:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776479.581336:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581341:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776479.581342:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581343:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.581344:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581345:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776479.581346:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776479.581348:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776479.581350:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776479.581351:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776479.581353:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776479.581358:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756160:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776479.581361:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776479.581363:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581367:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776479.581368:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776479.581370:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776479.581371:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776479.581373:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776479.581375:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756288:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776479.581377:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776479.581378:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776479.581379:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776479.581380:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776479.581382:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776479.581384:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776479.581386:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776479.581388:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776479.581390:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776479.581391:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099756288, portal 10 00000100:00000001:3.0:1775776479.581393:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776479.581394:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287608192 : -114841421943424 : ffff978d65d6ed80) 00000100:00000040:3.0:1775776479.581397:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581402:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776479.581403:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776479.581404:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776479.581406:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099756288, offset 0 00000400:00000200:3.0:1775776479.581408:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776479.581410:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.581412:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776479.581414:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776479.581416:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776479.581417:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776479.581419:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776479.581421:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4139:4139:256:4294967295] 192.168.201.139@tcp LPNI seq info [4139:4139:8:4294967295] 00000400:00000200:3.0:1775776479.581424:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.581428:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776479.581431:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.581433:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776479.581435:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776479.581437:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776479.581440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776479.581441:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776479.581444:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.581445:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776479.581447:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776479.581449:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776479.581451:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776479.581452:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776479.581453:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776479.581455:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581460:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776479.581462:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581465:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581466:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776479.581468:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776479.581470:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776479.581471:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776479.581473:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312867334 00000100:00000001:3.0:1775776479.581474:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312868597 00000100:00000001:3.0:1775776479.581475:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312867334 00000100:00000001:3.0:1775776479.581476:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312868597 00000100:00000001:3.0:1775776479.581478:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776479.581479:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581480:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776479.581483:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776479/real 1775776479] req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776479.581488:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776479.581489:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776479.581490:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776479.581491:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776479.581492:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776479.581493:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776479.581495:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776479.581500:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581503:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776479.581504:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581505:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.581506:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776479.581508:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581512:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776479.581514:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581518:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776479.581519:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581520:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.581521:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581522:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776479.581523:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776479.581525:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776479.581527:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776479.581529:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776479.581531:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776479.581536:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756288:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776479.581539:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581540:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776479.581542:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ce00 x1862033099756160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581545:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776479.581546:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776479.581547:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776479.581549:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776479.581550:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776479.581550:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776479.581552:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776479.581553:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38c00. 02000000:00000001:3.0:1775776479.581554:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776479.581555:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776479.581556:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ce00. 00000100:00000001:3.0:1775776479.581558:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776479.581559:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581560:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776479.581561:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ed80 x1862033099756288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581565:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776479.581566:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776479.581567:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776479.581568:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776479.581569:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776479.581570:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776479.581571:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776479.581571:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38400. 02000000:00000001:3.0:1775776479.581573:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776479.581574:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776479.581575:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ed80. 00000100:00000001:3.0:1775776479.581576:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776479.581577:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581579:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581581:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776479.581582:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581584:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776479.581585:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776479.581586:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776479.581588:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581592:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776479.581593:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776479.581595:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776479.581596:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776479.581597:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776479.581599:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756416:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776479.581601:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776479.581602:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776479.581603:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776479.581604:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776479.581605:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776479.581606:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776479.581609:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776479.581610:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776479.581612:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776479.581614:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099756416, portal 4 00000100:00000001:3.0:1775776479.581615:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776479.581616:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609984 : -114841421941632 : ffff978d65d6f480) 00000100:00000040:3.0:1775776479.581620:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581624:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776479.581625:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776479.581627:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776479.581629:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099756416, offset 0 00000400:00000200:3.0:1775776479.581631:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776479.581633:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.581635:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776479.581637:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776479.581639:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776479.581641:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776479.581642:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776479.581644:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4140:4140:256:4294967295] 192.168.201.139@tcp LPNI seq info [4140:4140:8:4294967295] 00000400:00000200:3.0:1775776479.581647:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.581651:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776479.581653:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.581655:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776479.581657:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776479.581659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776479.581662:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776479.581663:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776479.581666:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.581667:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776479.581669:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776479.581671:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776479.581673:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776479.581674:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776479.581675:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776479.581677:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581682:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776479.581684:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581688:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581689:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776479.581691:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776479.581693:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776479.581695:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776479.581696:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312867343 00000100:00000001:3.0:1775776479.581698:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312868597 00000100:00000001:3.0:1775776479.581699:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312867343 00000100:00000001:3.0:1775776479.581701:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312868597 00000100:00000001:3.0:1775776479.581702:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776479.581703:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581704:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776479.581706:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776479/real 1775776479] req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776479.581711:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776479.581711:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776479.581712:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776479.581713:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776479.581714:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776479.581716:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776479.581717:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776479.581723:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581726:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776479.581727:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581728:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.581729:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776479.581731:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581735:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776479.581737:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581742:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776479.581744:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581745:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.581746:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581747:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776479.581748:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776479.581749:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776479.581751:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776479.581752:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776479.581754:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776479.581759:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756416:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776479.581761:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776479.581763:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581768:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776479.581769:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776479.581772:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776479.581773:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776479.581775:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776479.581798:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756544:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776479.581802:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776479.581804:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776479.581805:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776479.581807:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776479.581811:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776479.581813:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776479.581816:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776479.581819:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776479.581822:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776479.581825:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099756544, portal 4 00000100:00000001:3.0:1775776479.581827:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776479.581829:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287596544 : -114841421955072 : ffff978d65d6c000) 00000100:00000040:3.0:1775776479.581835:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581842:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776479.581844:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776479.581847:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776479.581850:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099756544, offset 0 00000400:00000200:3.0:1775776479.581853:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776479.581856:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.581859:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776479.581861:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776479.581863:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776479.581865:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776479.581866:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776479.581868:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4141:4141:256:4294967295] 192.168.201.139@tcp LPNI seq info [4141:4141:8:4294967295] 00000400:00000200:3.0:1775776479.581871:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776479.581875:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776479.581877:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.581879:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776479.581882:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776479.581884:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776479.581887:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776479.581888:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776479.581891:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776479.581892:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776479.581894:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776479.581896:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776479.581898:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776479.581899:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776479.581901:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776479.581904:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581909:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776479.581911:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776504 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581915:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581916:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776479.581918:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776479.581920:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776479.581922:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776479.581923:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312867335 00000100:00000001:3.0:1775776479.581925:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312868597 00000100:00000001:3.0:1775776479.581926:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312867335 00000100:00000001:3.0:1775776479.581928:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312868597 00000100:00000001:3.0:1775776479.581929:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776479.581930:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.581931:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776479.581933:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776479/real 1775776479] req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776479.581938:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776479.581939:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776479.581940:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776479.581941:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776479.581942:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776479.581944:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776479.581946:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776479.581950:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581958:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776479.581959:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581960:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.581961:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776479.581964:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581967:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776479.581970:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581974:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776479.581975:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581976:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776479.581977:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581978:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776479.581979:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776479.581980:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776479.581982:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776479.581983:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776479.581986:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776479.581990:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756544:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776479.581992:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.581993:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776479.581995:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f480 x1862033099756416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.581999:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776479.582000:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776479.582002:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776479.582003:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776479.582004:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776479.582005:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776479.582007:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776479.582008:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39800. 02000000:00000001:3.0:1775776479.582009:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776479.582010:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776479.582011:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f480. 00000100:00000001:3.0:1775776479.582013:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776479.582013:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.582014:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776479.582016:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c000 x1862033099756544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776504 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776479.582021:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776479.582021:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776479.582022:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776479.582023:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776479.582024:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776479.582025:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776479.582026:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776479.582027:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39400. 02000000:00000001:3.0:1775776479.582028:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776479.582028:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776479.582029:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c000. 00000100:00000001:3.0:1775776479.582030:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776479.582031:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.582032:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776479.582034:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776479.582035:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776479.582036:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776479.582037:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776480.283897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776480.283902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776480.283904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776480.283908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776480.283912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776480.283917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776480.283919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776480.283922:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776480.283925:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776480.347899:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776480.347907:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776480.347909:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776480.347915:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776480.347919:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776480.347922:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776480.347928:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776480.347931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776480.347935:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776480.348020:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776480.348023:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776480.348026:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776480.859860:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312868597, at 4312868608 00000100:00000040:0.0:1775776480.859866:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312868597, at 4312868608 00000100:00000010:1.0:1775776480.859875:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c60. 00000100:00000010:0.0:1775776480.859876:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344360. 00000100:00000001:3.0:1775776481.307894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776481.307902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776481.307904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776481.307909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776481.307913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776481.307917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776481.307920:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776481.307922:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776481.307925:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776481.371876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776481.371885:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776481.371889:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776481.371896:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776481.371902:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776481.371907:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776481.371912:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776481.371916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776481.371924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776481.371973:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776481.371976:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776481.371979:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776482.331896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776482.331904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776482.331906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776482.331912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776482.331916:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776482.331919:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776482.331921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776482.331925:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776482.331928:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776482.395893:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776482.395899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776482.395903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776482.395907:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776482.395912:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776482.395917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776482.395922:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776482.395924:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776482.395926:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776482.395987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776482.395992:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776482.395996:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776483.355861:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776483.355869:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776483.355871:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776483.355877:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776483.355880:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776483.355885:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776483.355888:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776483.355890:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776483.355894:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776483.419874:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776483.419882:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776483.419885:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776483.419891:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776483.419897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776483.419902:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776483.419909:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776483.419914:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776483.419921:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776483.419983:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776483.419987:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776483.419991:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776484.379897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776484.379906:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776484.379908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776484.379914:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776484.379918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776484.379922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.379925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776484.379928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.379931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.443894:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776484.443902:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776484.443905:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776484.443912:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776484.443917:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776484.443922:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.443926:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776484.443929:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.443934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776484.443988:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776484.443991:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776484.443994:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776484.699902:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776484.699910:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776484.699912:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776484.699915:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776484.699918:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776484.699922:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18200 00000400:00000200:0.0:1775776484.699931:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776484.699935:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776484.699938:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776484.699940:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.699942:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776484.699944:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.699945:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776484.699948:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776484.699949:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776484.699951:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776484.699953:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776484.699954:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776484.699955:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776484.699957:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.699958:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776484.699959:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.699960:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776484.699962:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776484.699964:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776484.699965:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776484.699967:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776484.699969:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776484.699971:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.699972:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776484.699974:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776484.699981:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776484.699983:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776484.699986:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776484.699987:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.699989:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776484.699991:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776484.699994:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776484.700001:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776484.700006:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776484.700012:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776484.700018:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776484.700022:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776484.700024:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776484.700038:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776484.700040:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776484.700043:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776484.700044:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776484.700045:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776484.700047:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776484.700049:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18200 00000400:00000200:0.0:1775776484.700053:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776484.700055:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776484.700057:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776484.700061:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700063:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776484.700065:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700068:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776484.700070:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775776484.700071:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1775776484.700072:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776484.700075:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776484.700075:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1775776484.700087:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776484.700091:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776484.700091:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776484.700097:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776484.700102:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776484.700105:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776484.700107:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776484.700110:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000001:3.0:1775776484.700111:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776484.700114:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776484.700115:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776484.700118:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000001:3.0:1775776484.700118:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.700122:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776484.700125:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00100000:3.0:1775776484.700125:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099756672:192.168.201.139@tcp:250:kworker.0 00000100:00080000:0.0:1775776484.700131:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776484.700132:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776484.700133:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776484.700135:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776484.700135:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776484.700138:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776484.700140:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776484.700140:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775776484.700143:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776484.700144:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 00000100:00000001:0.0:1775776484.700145:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:3.0:1775776484.700148:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776484.700149:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776484.700152:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776484.700154:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776484.700155:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776484.700158:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000100:00000010:0.0:1775776484.700160:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776484.700163:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776484.700164:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:3.0:1775776484.700164:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 02000000:00000001:0.0:1775776484.700166:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.700168:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776484.700169:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776484.700171:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775776484.700172:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099756672, portal 25 00000100:00000001:3.0:1775776484.700174:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000010:0.0:1775776484.700175:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:3.0:1775776484.700175:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000100:00000001:0.0:1775776484.700179:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776484.700181:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776484.700186:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776484.700211:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776484.700214:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.700214:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776484.700221:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776484.700222:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776484.700228:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000001:0.0:1775776484.700230:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776484.700232:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:3.0:1775776484.700233:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099756672, offset 0 00000100:00080000:0.0:1775776484.700236:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776484.700238:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776484.700239:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776484.700242:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775776484.700244:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00080000:0.0:1775776484.700245:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18200 00000400:00000200:0.0:1775776484.700250:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776484.700252:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776484.700254:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1775776484.700255:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:0.0:1775776484.700258:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700260:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775776484.700263:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000040:0.0:1775776484.700265:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700267:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:3.0:1775776484.700268:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775776484.700269:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776484.700271:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776484.700272:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1775776484.700273:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000020:00000040:0.0:1775776484.700274:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776484.700276:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:3.0:1775776484.700278:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000020:00000001:0.0:1775776484.700280:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:3.0:1775776484.700282:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000040:0.0:1775776484.700287:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776484.700288:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4142:4142:256:4294967295] 192.168.201.139@tcp LPNI seq info [4142:4142:8:4294967295] 00000100:00000001:0.0:1775776484.700292:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775776484.700294:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:0.0:1775776484.700298:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700300:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776484.700305:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:3.0:1775776484.700307:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00080000:0.0:1775776484.700312:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000800:00000200:3.0:1775776484.700315:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700318:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775776484.700320:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 02000000:00000001:0.0:1775776484.700321:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776484.700325:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1775776484.700328:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776484.700329:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775776484.700332:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1775776484.700333:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776484.700336:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:3.1:1775776484.700337:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00000001:0.0:1775776484.700338:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.700341:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776484.700341:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000010:0.0:1775776484.700348:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000800:00000100:3.0:1775776484.700349:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000020:00000040:0.0:1775776484.700351:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000010:3.0:1775776484.700352:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 02000000:00000001:0.0:1775776484.700354:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776484.700356:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.700357:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776484.700358:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776484.700360:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776484.700360:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:0.0:1775776484.700364:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776484.700369:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776484.700369:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00100000:0.0:1775776484.700374:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775776484.700376:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776484.700380:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:0.0:1775776484.700382:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776484.700384:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.700386:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1775776484.700387:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776484.700393:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776484.700395:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776484.700398:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776484.700404:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776484.700406:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776484.700408:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776484.700413:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776484.700417:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1775776484.700418:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18200 00000100:00000040:3.0:1775776484.700423:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776484.700424:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776484.700432:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776484.700437:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776484.700444:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700449:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776484.700450:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.700452:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776484.700458:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776484.700461:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700465:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:3.0:1775776484.700466:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:0.0:1775776484.700467:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776484.700469:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776484.700471:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776484.700471:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312873717 00000020:00000040:0.0:1775776484.700472:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776484.700475:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776484.700477:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776484.700479:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000010:3.0:1775776484.700479:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503441e0. 00000100:00000001:0.0:1775776484.700481:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776484.700484:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700486:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776484.700487:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776484.700489:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776484.700489:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776484.700493:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776484.700495:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776484.700497:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776484.700498:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312873717 02000000:00000001:0.0:1775776484.700499:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.700501:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776484.700504:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000010:3.0:1775776484.700504:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503442a0. 00000008:00000001:0.0:1775776484.700506:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.700509:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776484.700510:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776484.700513:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000100:00000001:3.0:1775776484.700514:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1775776484.700516:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00000001:3.0:1775776484.700518:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776484.700519:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776484.700522:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.700523:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775776484.700523:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1775776484.700526:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776484.700527:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776484.700530:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776484/real 1775776484] req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775776484.700531:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776484.700535:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776484.700542:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1775776484.700547:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1775776484.700548:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776484.700551:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1775776484.700551:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1775776484.700555:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775776484.700555:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776484.700561:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000100:00000001:0.0:1775776484.700563:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776484.700564:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00080000:0.0:1775776484.700566:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775776484.700568:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1775776484.700569:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776484.700572:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776484.700573:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776484.700576:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:3.0:1775776484.700576:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776484.700580:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18200 00000400:00000200:0.0:1775776484.700587:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776484.700592:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776484.700594:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000200:3.0:1775776484.700595:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776484.700598:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700601:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776484.700605:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700608:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776484.700612:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776484.700614:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000100:00000001:3.0:1775776484.700614:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1775776484.700617:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776484.700617:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775776484.700618:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776484.700622:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:3.0:1775776484.700623:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1775776484.700624:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:3.0:1775776484.700626:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776484.700627:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700631:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776484.700635:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00100000:3.0:1775776484.700635:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776484.700636:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776484.700640:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776484.700644:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776484.700647:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776484.700650:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776484.700652:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776484.700656:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.700656:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776484.700657:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776484.700660:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776484.700662:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.700664:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776484.700664:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776484.700667:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776484.700670:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776484.700674:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776484.700676:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776484.700679:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776484.700682:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775776484.700682:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1775776484.700683:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776484.700688:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:3.0:1775776484.700688:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776484.700692:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.700692:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.700694:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776484.700698:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776484.700699:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776484.700703:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776484.700705:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776484.700708:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776484.700709:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1775776484.700712:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776484.700717:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776484.700721:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1775776484.700727:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776484.700727:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776484.700731:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18210) 00000100:00100000:3.0:1775776484.700740:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099756672:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776484.700746:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.700748:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.700752:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099756672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.700758:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776484.700760:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776484.700762:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776484.700774:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776484.700805:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776484.700806:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776484.700808:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776484.700810:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759dc00. 02000000:00000001:3.0:1775776484.700815:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776484.700817:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776484.700818:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:3.0:1775776484.700821:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776484.700822:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.700824:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.700830:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776484.700832:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.700834:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776484.700835:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776484.700836:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776484.700840:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.700847:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776484.700848:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776484.700850:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776484.700852:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776484.700854:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776484.700856:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776484.700859:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776484.700861:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776484.700862:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776484.700864:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776484.700867:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776484.700868:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776484.700872:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776484.700875:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776484.700878:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776484.700881:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099756800, portal 10 00000100:00000001:3.0:1775776484.700882:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776484.700884:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:3.0:1775776484.700889:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.700895:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776484.700898:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776484.700900:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776484.700902:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099756800, offset 0 00000400:00000200:3.0:1775776484.700906:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776484.700909:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776484.700912:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776484.700915:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776484.700917:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776484.700919:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776484.700922:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776484.700924:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4143:4143:256:4294967295] 192.168.201.139@tcp LPNI seq info [4143:4143:8:4294967295] 00000400:00000200:3.0:1775776484.700928:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776484.700934:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776484.700938:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776484.700941:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776484.700944:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776484.700947:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775776484.700951:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776484.700953:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775776484.700956:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776484.700957:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776484.700960:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776484.700963:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776484.700965:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776484.700967:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776484.700969:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776484.700973:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.700980:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.700982:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.700989:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.700990:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776484.700993:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776484.700997:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776484.700999:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776484.701001:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312872453 00000100:00000001:3.0:1775776484.701004:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312873717 00000100:00000001:3.0:1775776484.701006:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312872453 00000100:00000001:3.0:1775776484.701008:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312873717 00000100:00000001:3.0:1775776484.701009:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776484.701011:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.701013:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776484.701016:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776484/real 1775776484] req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776484.701023:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776484.701024:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776484.701026:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776484.701028:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776484.701029:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776484.701031:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776484.701034:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776484.701042:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701048:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776484.701049:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701051:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.701052:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776484.701055:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701060:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776484.701063:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701069:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776484.701071:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701072:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.701073:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701075:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776484.701089:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776484.701092:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776484.701095:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776484.701097:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776484.701101:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776484.701108:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776484.701112:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776484.701115:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701121:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776484.701122:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776484.701124:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776484.701125:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776484.701127:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776484.701130:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776484.701133:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776484.701134:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776484.701135:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776484.701137:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776484.701140:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776484.701142:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776484.701145:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776484.701147:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776484.701150:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776484.701153:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099756928, portal 10 00000100:00000001:3.0:1775776484.701154:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776484.701155:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:3.0:1775776484.701160:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701166:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776484.701168:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776484.701170:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776484.701172:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099756928, offset 0 00000400:00000200:3.0:1775776484.701175:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776484.701179:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776484.701182:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776484.701184:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776484.701187:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776484.701189:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776484.701191:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776484.701194:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4144:4144:256:4294967295] 192.168.201.139@tcp LPNI seq info [4144:4144:8:4294967295] 00000400:00000200:3.0:1775776484.701198:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776484.701204:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776484.701208:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776484.701211:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776484.701215:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776484.701217:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775776484.701221:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776484.701223:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775776484.701226:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776484.701228:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776484.701231:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776484.701234:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776484.701237:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776484.701239:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776484.701241:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776484.701244:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701252:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.701254:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701260:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.701261:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776484.701264:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776484.701267:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776484.701269:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776484.701271:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312872454 00000100:00000001:3.0:1775776484.701273:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312873717 00000100:00000001:3.0:1775776484.701275:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312872454 00000100:00000001:3.0:1775776484.701277:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312873717 00000100:00000001:3.0:1775776484.701279:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776484.701281:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.701283:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776484.701287:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776484/real 1775776484] req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776484.701294:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776484.701295:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776484.701296:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776484.701299:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776484.701300:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776484.701302:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776484.701305:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776484.701312:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701318:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776484.701319:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701321:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.701322:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776484.701325:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701331:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776484.701335:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701341:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776484.701343:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701345:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.701346:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701348:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776484.701349:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776484.701351:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776484.701354:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776484.701356:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776484.701360:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776484.701366:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099756928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776484.701370:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776484.701373:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701379:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776484.701380:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776484.701383:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776484.701384:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776484.701386:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776484.701389:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776484.701391:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776484.701393:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776484.701394:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776484.701396:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776484.701399:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776484.701401:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776484.701403:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776484.701406:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776484.701409:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776484.701411:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099757056, portal 4 00000100:00000001:3.0:1775776484.701413:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776484.701415:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:3.0:1775776484.701419:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701425:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776484.701427:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776484.701429:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776484.701431:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099757056, offset 0 00000400:00000200:3.0:1775776484.701435:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776484.701438:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776484.701441:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776484.701444:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776484.701446:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776484.701449:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776484.701451:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776484.701454:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4145:4145:256:4294967295] 192.168.201.139@tcp LPNI seq info [4145:4145:8:4294967295] 00000400:00000200:3.0:1775776484.701458:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776484.701463:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776484.701468:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776484.701472:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776484.701475:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776484.701478:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775776484.701482:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776484.701484:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775776484.701499:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776484.701501:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776484.701504:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776484.701507:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776484.701511:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776484.701512:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776484.701515:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776484.701518:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701525:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.701528:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701533:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.701535:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776484.701538:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776484.701541:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776484.701544:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776484.701545:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312872463 00000100:00000001:3.0:1775776484.701548:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312873717 00000100:00000001:3.0:1775776484.701550:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312872463 00000100:00000001:3.0:1775776484.701552:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312873717 00000100:00000001:3.0:1775776484.701553:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776484.701555:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.701557:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776484.701561:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776484/real 1775776484] req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776484.701568:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776484.701569:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776484.701571:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776484.701573:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776484.701575:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776484.701577:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776484.701579:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776484.701587:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701593:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776484.701595:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701596:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.701598:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776484.701601:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701607:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776484.701610:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701616:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776484.701618:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701620:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.701621:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701623:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776484.701624:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776484.701627:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776484.701630:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776484.701632:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776484.701635:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776484.701642:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776484.701645:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776484.701648:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701654:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776484.701655:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776484.701657:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776484.701659:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776484.701660:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776484.701663:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776484.701666:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776484.701667:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776484.701669:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776484.701670:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776484.701674:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776484.701676:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776484.701679:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776484.701681:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776484.701683:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776484.701686:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099757184, portal 4 00000100:00000001:3.0:1775776484.701687:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776484.701688:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757056 : -114841967794560 : ffff978d454de680) 00000100:00000040:3.0:1775776484.701693:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701699:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776484.701701:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776484.701703:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776484.701706:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099757184, offset 0 00000400:00000200:3.0:1775776484.701710:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776484.701713:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776484.701716:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776484.701719:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776484.701721:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776484.701724:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776484.701726:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776484.701728:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4146:4146:256:4294967295] 192.168.201.139@tcp LPNI seq info [4146:4146:8:4294967295] 00000400:00000200:3.0:1775776484.701732:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776484.701738:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776484.701743:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776484.701746:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776484.701749:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776484.701751:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775776484.701755:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776484.701757:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1775776484.701760:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776484.701762:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776484.701764:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776484.701767:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776484.701769:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776484.701771:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776484.701772:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776484.701775:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701801:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.701804:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776509 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701810:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.701811:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776484.701814:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776484.701816:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776484.701819:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776484.701821:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312872455 00000100:00000001:3.0:1775776484.701823:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312873717 00000100:00000001:3.0:1775776484.701825:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312872455 00000100:00000001:3.0:1775776484.701827:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312873717 00000100:00000001:3.0:1775776484.701829:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776484.701831:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.701833:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776484.701836:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776484/real 1775776484] req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776484.701843:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776484.701845:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776484.701846:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776484.701848:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776484.701850:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776484.701851:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776484.701854:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776484.701862:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701869:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776484.701870:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701872:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.701873:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776484.701876:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701882:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776484.701886:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701892:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776484.701894:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701896:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776484.701897:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701899:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776484.701900:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776484.701903:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776484.701906:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776484.701908:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776484.701911:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776484.701919:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776484.701922:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701924:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.701927:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099756800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701932:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776484.701934:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776484.701935:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776484.701938:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776484.701939:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776484.701940:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776484.701942:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776484.701943:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d000. 02000000:00000001:3.0:1775776484.701945:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776484.701946:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776484.701948:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000100:00000001:3.0:1775776484.701950:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776484.701951:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701952:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.701955:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099756928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701965:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776484.701966:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776484.701967:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776484.701969:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776484.701970:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776484.701971:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776484.701973:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776484.701974:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:3.0:1775776484.701975:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776484.701976:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776484.701977:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:3.0:1775776484.701979:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776484.701980:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.701981:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.701983:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099757056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.701989:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776484.701990:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776484.701991:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776484.701993:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776484.701994:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776484.701995:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776484.701996:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776484.701998:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:3.0:1775776484.702000:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776484.702001:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776484.702002:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:3.0:1775776484.702004:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776484.702005:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.702006:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776484.702009:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de680 x1862033099757184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776509 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776484.702014:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776484.702015:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776484.702017:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776484.702019:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776484.702020:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776484.702021:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776484.702023:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776484.702024:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:3.0:1775776484.702026:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776484.702027:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776484.702029:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de680. 00000100:00000001:3.0:1775776484.702031:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776484.702032:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.702034:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776484.702037:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776484.702038:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776484.702040:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776484.702041:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776485.403875:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776485.403882:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776485.403884:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776485.403889:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776485.403893:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776485.403896:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776485.403899:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776485.403902:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776485.403904:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776485.467860:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776485.467867:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776485.467869:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776485.467875:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776485.467880:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776485.467884:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776485.467887:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776485.467890:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776485.467893:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776485.468017:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776485.468020:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776485.468022:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776485.979915:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312873717, at 4312873728 00000100:00000040:0.0:1775776485.979916:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312873717, at 4312873728 00000100:00000010:0.0:1775776485.979937:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000100:00000010:1.0:1775776485.979940:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503442a0. 00000100:00000001:3.0:1775776486.427887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776486.427894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776486.427896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776486.427900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776486.427905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776486.427908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776486.427911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776486.427913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776486.427915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776486.491925:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776486.491933:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776486.491935:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776486.491942:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776486.491947:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776486.491961:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776486.491968:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776486.491972:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776486.491978:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776486.492073:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776486.492079:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776486.492083:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776487.451892:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776487.451897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776487.451899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776487.451903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776487.451906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776487.451911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776487.451913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776487.451915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776487.451918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776487.515894:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776487.515901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776487.515902:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776487.515908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776487.515912:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776487.515916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776487.515921:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776487.515926:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776487.515933:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776487.516008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776487.516011:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776487.516014:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776488.475864:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776488.475872:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776488.475874:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776488.475878:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776488.475882:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776488.475886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776488.475889:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776488.475891:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776488.475893:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776488.539901:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776488.539907:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776488.539909:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776488.539913:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776488.539917:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776488.539925:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776488.539928:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776488.539931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776488.539935:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776488.540027:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776488.540030:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776488.540032:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776489.499900:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776489.499906:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776489.499908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776489.499913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776489.499916:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776489.499920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.499922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776489.499924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.499927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.563870:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776489.563884:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776489.563889:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776489.563897:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776489.563902:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776489.563907:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.563911:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776489.563916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.563919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776489.564013:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776489.564017:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776489.564020:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776489.820156:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776489.820161:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776489.820165:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776489.820174:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820190:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776489.820193:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.820209:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776489.820218:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776489.820224:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776489.820232:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099757312:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776489.820240:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776489.820246:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776489.820249:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.820255:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776489.820260:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776489.820264:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776489.820274:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776489.820279:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776489.820288:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776489.820297:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099757312, portal 25 00000100:00000001:3.0:1775776489.820302:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776489.820306:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288717824 : -114841420833792 : ffff978d65e7dc00) 00000100:00000040:3.0:1775776489.820317:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820335:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776489.820341:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776489.820346:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776489.820360:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099757312, offset 0 00000400:00000200:3.0:1775776489.820372:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776489.820383:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.820393:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776489.820400:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776489.820406:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776489.820411:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776489.820416:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776489.820425:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4147:4147:256:4294967295] 192.168.201.139@tcp LPNI seq info [4147:4147:8:4294967295] 00000400:00000200:3.0:1775776489.820431:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.820445:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776489.820462:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.820469:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776489.820481:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776489.820487:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776489.820500:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776489.820506:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776489.820517:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.820521:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776489.820528:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776489.820534:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776489.820539:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776489.820543:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776489.820549:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776489.820557:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820585:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.820593:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820612:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.820618:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776489.820625:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776489.820641:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776489.820646:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312878837 00000100:00000010:3.0:1775776489.820652:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503445a0. 00000100:00000001:3.0:1775776489.820660:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776489.820670:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312878837 00000100:00000010:3.0:1775776489.820675:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503447e0. 00000100:00000001:3.0:1775776489.820680:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776489.820686:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776489.820692:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.820695:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776489.820703:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776489/real 1775776489] req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776489.820739:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776489.820742:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776489.820746:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776489.820750:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776489.820752:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776489.820754:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776489.820759:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776489.820768:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820775:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776489.820808:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.820812:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.820813:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776489.820818:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820825:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776489.820829:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820836:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776489.820839:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.820841:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.820842:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.820844:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776489.820846:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776489.820855:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776489.820859:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776489.820861:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776489.820865:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776489.820873:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099757312:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776489.820878:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.820880:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.820882:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7dc00 x1862033099757312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820889:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776489.820891:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776489.820894:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776489.820897:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776489.820898:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776489.820900:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776489.820902:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776489.820904:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:3.0:1775776489.820907:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776489.820909:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776489.820911:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7dc00. 00000100:00000001:3.0:1775776489.820914:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776489.820915:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.820917:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.820924:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776489.820925:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.820927:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776489.820929:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776489.820930:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776489.820933:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.820940:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776489.820941:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.820943:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776489.820945:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776489.820946:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776489.820949:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757440:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776489.820953:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776489.820955:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776489.820956:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.820958:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776489.820961:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776489.820962:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776489.820966:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776489.820968:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776489.820971:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776489.820974:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099757440, portal 10 00000100:00000001:3.0:1775776489.820976:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776489.820977:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716928 : -114841420834688 : ffff978d65e7d880) 00000100:00000040:3.0:1775776489.820983:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821002:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776489.821004:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776489.821007:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776489.821009:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099757440, offset 0 00000400:00000200:3.0:1775776489.821013:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776489.821017:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.821021:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776489.821024:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776489.821026:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776489.821028:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776489.821030:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776489.821033:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4148:4148:256:4294967295] 192.168.201.139@tcp LPNI seq info [4148:4148:8:4294967295] 00000400:00000200:3.0:1775776489.821037:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.821043:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776489.821048:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.821051:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776489.821055:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776489.821058:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776489.821063:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776489.821065:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776489.821069:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.821070:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776489.821073:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776489.821076:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776489.821080:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776489.821081:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776489.821083:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776489.821086:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821093:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.821096:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821102:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.821104:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776489.821107:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776489.821110:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776489.821113:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776489.821115:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312877573 00000100:00000001:3.0:1775776489.821118:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312878837 00000100:00000001:3.0:1775776489.821120:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312877573 00000100:00000001:3.0:1775776489.821122:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312878837 00000100:00000001:3.0:1775776489.821124:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776489.821127:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.821128:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776489.821132:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776489/real 1775776489] req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776489.821139:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776489.821140:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776489.821143:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776489.821144:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776489.821146:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776489.821148:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776489.821151:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776489.821159:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821165:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776489.821166:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821168:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.821169:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776489.821173:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821179:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776489.821182:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821189:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776489.821191:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821192:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.821194:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821195:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776489.821197:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776489.821200:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776489.821203:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776489.821205:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776489.821208:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776489.821216:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757440:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776489.821220:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776489.821223:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821229:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776489.821230:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.821232:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776489.821234:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776489.821236:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776489.821239:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757568:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776489.821242:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776489.821244:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776489.821245:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.821247:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776489.821250:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776489.821252:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776489.821255:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776489.821258:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776489.821260:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776489.821263:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099757568, portal 10 00000100:00000001:3.0:1775776489.821265:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776489.821266:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724096 : -114841420827520 : ffff978d65e7f480) 00000100:00000040:3.0:1775776489.821271:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821277:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776489.821279:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776489.821281:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776489.821284:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099757568, offset 0 00000400:00000200:3.0:1775776489.821287:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776489.821290:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.821293:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776489.821296:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776489.821299:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776489.821301:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776489.821303:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776489.821306:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4149:4149:256:4294967295] 192.168.201.139@tcp LPNI seq info [4149:4149:8:4294967295] 00000400:00000200:3.0:1775776489.821310:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.821316:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776489.821320:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.821323:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776489.821326:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776489.821329:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776489.821333:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776489.821335:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776489.821339:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.821340:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776489.821343:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776489.821346:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776489.821349:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776489.821350:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776489.821353:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776489.821356:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821363:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.821366:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821372:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.821373:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776489.821376:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776489.821379:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776489.821382:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776489.821384:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312877574 00000100:00000001:3.0:1775776489.821386:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312878837 00000100:00000001:3.0:1775776489.821388:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312877574 00000100:00000001:3.0:1775776489.821390:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312878837 00000100:00000001:3.0:1775776489.821392:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776489.821394:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.821396:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776489.821399:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776489/real 1775776489] req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776489.821407:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776489.821408:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776489.821410:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776489.821412:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776489.821413:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776489.821415:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776489.821418:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776489.821426:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821433:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776489.821434:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821436:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.821437:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776489.821440:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821446:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776489.821449:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821456:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776489.821458:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821460:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.821461:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821463:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776489.821464:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776489.821466:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776489.821470:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776489.821472:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776489.821475:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776489.821482:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757568:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776489.821485:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776489.821488:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821494:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776489.821495:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.821497:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776489.821499:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776489.821501:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776489.821504:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757696:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776489.821506:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776489.821508:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776489.821510:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.821511:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776489.821514:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776489.821517:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776489.821520:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776489.821522:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776489.821525:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776489.821528:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099757696, portal 4 00000100:00000001:3.0:1775776489.821530:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776489.821531:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288723200 : -114841420828416 : ffff978d65e7f100) 00000100:00000040:3.0:1775776489.821536:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821542:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776489.821544:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776489.821547:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776489.821549:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099757696, offset 0 00000400:00000200:3.0:1775776489.821554:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776489.821557:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.821560:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776489.821563:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776489.821565:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776489.821567:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776489.821570:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776489.821573:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4150:4150:256:4294967295] 192.168.201.139@tcp LPNI seq info [4150:4150:8:4294967295] 00000400:00000200:3.0:1775776489.821577:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.821583:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776489.821587:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.821590:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776489.821593:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776489.821596:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776489.821600:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776489.821602:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776489.821606:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.821608:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776489.821610:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776489.821614:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776489.821616:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776489.821618:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776489.821620:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776489.821623:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821629:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.821632:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821643:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.821645:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776489.821648:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776489.821650:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776489.821654:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776489.821655:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312877583 00000100:00000001:3.0:1775776489.821657:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312878837 00000100:00000001:3.0:1775776489.821659:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312877583 00000100:00000001:3.0:1775776489.821661:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312878837 00000100:00000001:3.0:1775776489.821663:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776489.821665:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.821667:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776489.821670:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776489/real 1775776489] req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776489.821678:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776489.821679:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776489.821681:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776489.821683:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776489.821684:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776489.821686:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776489.821689:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776489.821697:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821703:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776489.821705:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821706:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.821708:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776489.821711:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821717:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776489.821721:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821728:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776489.821730:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821732:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.821733:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.821734:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776489.821736:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776489.821738:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776489.821741:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776489.821743:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776489.821747:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776489.821754:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757696:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776489.821758:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776489.821761:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821767:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776489.821768:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.821770:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776489.821772:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776489.821774:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776489.821776:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757824:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776489.821798:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776489.821800:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776489.821802:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776489.821803:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776489.821807:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776489.821808:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776489.821812:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776489.821814:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776489.821817:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776489.821819:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099757824, portal 4 00000100:00000001:3.0:1775776489.821821:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776489.821822:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716032 : -114841420835584 : ffff978d65e7d500) 00000100:00000040:3.0:1775776489.821828:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821834:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776489.821836:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776489.821838:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776489.821840:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099757824, offset 0 00000400:00000200:3.0:1775776489.821844:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776489.821848:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.821851:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776489.821853:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776489.821856:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776489.821858:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776489.821860:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776489.821863:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4151:4151:256:4294967295] 192.168.201.139@tcp LPNI seq info [4151:4151:8:4294967295] 00000400:00000200:3.0:1775776489.821867:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776489.821873:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776489.821877:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.821880:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776489.821883:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776489.821886:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776489.821890:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776489.821892:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776489.821896:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776489.821898:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776489.821901:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776489.821904:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776489.821906:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776489.821908:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776489.821910:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776489.821913:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821921:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.821924:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776514 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.821930:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.821932:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776489.821934:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776489.821937:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776489.821940:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776489.821941:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312877575 00000100:00000001:3.0:1775776489.821944:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312878837 00000100:00000001:3.0:1775776489.821945:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312877575 00000100:00000001:3.0:1775776489.821948:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312878837 00000100:00000001:3.0:1775776489.821949:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776489.821951:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.821953:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776489.821956:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776489/real 1775776489] req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776489.821963:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776489.821964:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776489.821966:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776489.821968:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776489.821969:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776489.821972:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776489.821974:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776489.821983:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.822005:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776489.822006:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822007:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.822008:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776489.822012:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.822018:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776489.822022:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.822028:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776489.822030:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822032:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776489.822033:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822035:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776489.822037:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776489.822039:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776489.822042:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776489.822044:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776489.822048:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776489.822056:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099757824:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776489.822059:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822061:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.822065:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d880 x1862033099757440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.822071:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776489.822072:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776489.822074:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776489.822077:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776489.822078:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776489.822079:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776489.822082:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776489.822083:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504400. 02000000:00000001:3.0:1775776489.822086:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776489.822087:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776489.822088:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d880. 00000100:00000001:3.0:1775776489.822090:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776489.822092:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822093:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.822097:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f480 x1862033099757568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.822103:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776489.822104:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776489.822105:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776489.822107:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776489.822108:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776489.822109:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776489.822111:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776489.822112:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:3.0:1775776489.822115:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776489.822116:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776489.822117:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f480. 00000100:00000001:3.0:1775776489.822119:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776489.822120:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822122:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.822124:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f100 x1862033099757696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.822131:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776489.822132:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776489.822133:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776489.822135:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776489.822136:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776489.822138:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776489.822139:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776489.822140:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505c00. 02000000:00000001:3.0:1775776489.822142:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776489.822143:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776489.822144:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f100. 00000100:00000001:3.0:1775776489.822146:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776489.822147:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822148:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776489.822151:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d500 x1862033099757824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776514 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776489.822157:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776489.822158:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776489.822159:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776489.822161:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776489.822162:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776489.822163:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776489.822164:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776489.822165:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:3.0:1775776489.822167:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776489.822168:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776489.822169:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d500. 00000100:00000001:3.0:1775776489.822170:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776489.822171:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822173:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776489.822176:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776489.822178:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776489.822180:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776489.822181:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776490.523922:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776490.523930:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776490.523932:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776490.523937:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776490.523942:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776490.523947:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776490.523949:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776490.523951:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776490.523954:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776490.587842:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776490.587848:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776490.587850:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776490.587854:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776490.587857:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776490.587860:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776490.587863:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776490.587865:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776490.587868:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776490.587934:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776490.587937:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776490.587939:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776491.099875:0:314596:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312878837, at 4312878848 00000100:00000040:1.0:1775776491.099875:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312878837, at 4312878848 00000100:00000010:0.0:1775776491.099889:0:314596:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503445a0. 00000100:00000010:1.0:1775776491.099891:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503447e0. 00000100:00000001:3.0:1775776491.548128:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776491.548137:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776491.548139:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776491.548144:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776491.548147:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776491.548151:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776491.548153:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776491.548155:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776491.548158:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776491.611847:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776491.611853:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776491.611855:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776491.611859:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776491.611862:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776491.611864:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776491.611867:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776491.611869:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776491.611872:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776491.611930:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776491.611934:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776491.611937:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776492.572277:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776492.572285:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776492.572287:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776492.572298:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776492.572303:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776492.572306:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776492.572310:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776492.572313:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776492.572316:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776492.635875:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776492.635882:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776492.635885:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776492.635891:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776492.635897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776492.635902:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776492.635906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776492.635908:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776492.635911:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776492.635969:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776492.635973:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776492.635976:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776493.595884:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776493.595894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776493.595897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776493.595903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776493.595907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776493.595912:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776493.595916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776493.595918:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776493.596064:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776493.659853:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776493.659860:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776493.659862:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776493.659866:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776493.659871:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776493.659875:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776493.659878:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776493.659881:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776493.659884:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776493.659953:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776493.659957:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776493.659960:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776494.620021:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776494.620027:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776494.620029:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776494.620034:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776494.620038:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776494.620041:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.620044:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776494.620046:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.620049:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.683876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776494.683881:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776494.683883:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776494.683887:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776494.683890:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776494.683893:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.683895:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776494.683896:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.683898:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776494.683980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776494.683983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776494.683987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776494.939932:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776494.939947:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776494.939950:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776494.939955:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776494.939959:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776494.939966:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18210 00000400:00000200:1.0:1775776494.939977:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776494.939996:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776494.939998:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776494.940001:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940003:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940005:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940006:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776494.940009:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776494.940011:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776494.940012:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776494.940014:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776494.940015:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776494.940017:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776494.940019:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940020:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940022:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940023:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776494.940025:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776494.940027:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940029:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776494.940031:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776494.940034:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776494.940037:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940039:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776494.940041:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776494.940050:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776494.940054:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776494.940057:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776494.940059:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940062:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776494.940065:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776494.940069:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776494.940076:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776494.940082:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776494.940091:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776494.940101:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776494.940105:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776494.940110:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776494.940133:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776494.940136:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776494.940140:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776494.940141:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776494.940142:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776494.940144:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776494.940146:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18210 00000400:00000200:1.0:1775776494.940149:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776494.940151:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776494.940153:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776494.940155:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940157:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940159:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940161:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776494.940162:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776494.940163:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776494.940164:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776494.940165:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776494.940167:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776494.940167:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1775776494.940168:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776494.940172:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776494.940172:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776494.940175:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776494.940175:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776494.940177:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940180:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776494.940182:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000040:3.0:1775776494.940183:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776494.940186:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940190:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776494.940193:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775776494.940197:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1775776494.940198:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776494.940199:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776494.940203:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.940204:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775776494.940205:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:3.0:1775776494.940208:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:1.0:1775776494.940210:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1775776494.940210:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775776494.940214:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940217:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776494.940217:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099757952:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776494.940222:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1775776494.940223:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776494.940225:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776494.940226:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776494.940229:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776494.940229:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776494.940231:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.940232:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775776494.940233:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776494.940234:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776494.940236:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776494.940237:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000010:1.0:1775776494.940239:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 02000000:00000001:3.0:1775776494.940239:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940243:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.940245:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00100000:1.0:1775776494.940249:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1775776494.940249:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000100:00000001:1.0:1775776494.940257:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1775776494.940257:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000001:1.0:1775776494.940259:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776494.940262:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776494.940266:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099757952, portal 25 00000100:00000001:1.0:1775776494.940269:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.940270:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1775776494.940271:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776494.940274:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776494.940274:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776494.940275:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776494.940277:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776494.940279:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18210 00000100:00000001:3.0:1775776494.940281:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609088 : -114841421942528 : ffff978d65d6f100) 00000400:00000200:1.0:1775776494.940286:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776494.940291:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000040:3.0:1775776494.940291:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776494.940294:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776494.940301:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940306:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940310:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940315:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776494.940320:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775776494.940320:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1775776494.940323:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776494.940326:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:3.0:1775776494.940327:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000020:00000040:1.0:1775776494.940328:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000400:00000010:3.0:1775776494.940331:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000020:00000001:1.0:1775776494.940332:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776494.940336:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000200:3.0:1775776494.940337:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099757952, offset 0 00000100:00000040:1.0:1775776494.940340:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940342:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940343:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940345:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776494.940349:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:3.0:1775776494.940349:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00080000:1.0:1775776494.940353:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940355:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776494.940356:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776494.940358:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775776494.940361:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:1.0:1775776494.940362:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940364:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776494.940366:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776494.940368:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.940369:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:1.0:1775776494.940372:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776494.940375:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776494.940376:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776494.940378:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776494.940379:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940380:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1775776494.940381:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1775776494.940382:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776494.940385:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.940387:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 02000000:00000010:1.0:1775776494.940389:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000400:00000200:3.0:1775776494.940392:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775776494.940393:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.940396:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00100000:1.0:1775776494.940400:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1775776494.940401:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4152:4152:256:4294967295] 192.168.201.139@tcp LPNI seq info [4152:4152:8:4294967295] 00000400:00000200:3.0:1775776494.940410:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:1.0:1775776494.940414:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776494.940416:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776494.940419:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776494.940430:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.940430:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00080000:1.0:1775776494.940435:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776494.940438:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776494.940439:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776494.940441:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.0:1775776494.940442:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940443:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776494.940447:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18210 00000800:00000010:3.0:1775776494.940450:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000400:00000200:1.0:1775776494.940451:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776494.940455:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776494.940457:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776494.940461:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:3.0:1775776494.940462:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776494.940463:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940467:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940468:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000800:00000200:3.0:1775776494.940468:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1775776494.940472:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776494.940474:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776494.940475:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776494.940477:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776494.940479:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000800:00000200:3.1:1775776494.940479:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775776494.940481:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776494.940486:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:3.1:1775776494.940486:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1775776494.940489:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940492:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940494:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776494.940495:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776494.940497:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940498:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776494.940499:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776494.940501:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776494.940502:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940504:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000100:3.0:1775776494.940505:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000008:00100000:1.0:1775776494.940506:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776494.940509:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775776494.940509:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000100:00000001:1.0:1775776494.940512:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776494.940515:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776494.940517:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000400:00000200:3.0:1775776494.940518:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1775776494.940520:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776494.940523:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940526:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1775776494.940528:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1775776494.940529:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776494.940532:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.940535:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 02000000:00000010:1.0:1775776494.940537:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000400:00000010:3.0:1775776494.940538:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:1.0:1775776494.940543:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.940544:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776494.940549:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776494.940552:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776494.940563:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775776494.940566:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776494.940570:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776494.940570:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776494.940576:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776494.940582:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776494.940585:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776494.940591:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776494.940593:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776494.940595:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776494.940599:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776494.940603:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18210 00000400:00000200:1.0:1775776494.940610:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776494.940612:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776494.940613:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:3.0:1775776494.940615:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1775776494.940616:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776494.940619:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940620:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940622:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776494.940623:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940624:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776494.940627:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776494.940628:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776494.940629:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776494.940631:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000400:00000200:3.0:1775776494.940632:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:1.0:1775776494.940633:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776494.940636:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:3.0:1775776494.940637:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312883957 00000100:00000040:1.0:1775776494.940639:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940641:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776494.940643:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940644:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000010:3.0:1775776494.940645:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344600. 00000020:00000040:1.0:1775776494.940646:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776494.940650:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776494.940652:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776494.940654:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775776494.940654:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1775776494.940657:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776494.940658:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940660:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776494.940663:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776494.940666:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940668:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776494.940671:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776494.940674:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:3.0:1775776494.940675:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312883957 02000000:00000001:1.0:1775776494.940676:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776494.940678:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940680:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:3.0:1775776494.940680:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344ae0. 02000000:00000001:1.0:1775776494.940683:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775776494.940685:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1775776494.940686:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.940690:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1775776494.940692:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:3.0:1775776494.940695:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776494.940696:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.940699:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776494.940703:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775776494.940707:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776494/real 1775776494] req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776494.940729:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1775776494.940729:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1775776494.940732:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1775776494.940732:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776494.940734:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000100:00000040:1.0:1775776494.940738:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775776494.940738:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776494.940741:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776494.940745:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1775776494.940750:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775776494.940753:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776494.940755:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18220) 00000100:00000200:3.0:1775776494.940764:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.940772:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776494.940774:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.940776:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.940800:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776494.940804:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.940809:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776494.940812:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.940816:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776494.940818:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.940819:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.940819:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.940821:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776494.940822:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776494.940824:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776494.940826:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776494.940828:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776494.940830:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776494.940835:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099757952:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776494.940838:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.940839:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.940840:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f100 x1862033099757952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.940844:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776494.940845:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776494.940847:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776494.940849:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776494.940850:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776494.940851:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776494.940853:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776494.940854:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b800. 02000000:00000001:3.0:1775776494.940857:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776494.940858:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776494.940859:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f100. 00000100:00000001:3.0:1775776494.940861:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776494.940862:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.940863:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.940870:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776494.940872:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.940874:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776494.940875:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776494.940876:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776494.940879:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.940885:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776494.940886:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.940888:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776494.940890:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776494.940908:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776494.940912:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758080:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776494.940915:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776494.940917:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776494.940918:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.940920:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776494.940923:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776494.940925:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.940928:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776494.940931:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776494.940933:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776494.940935:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099758080, portal 10 00000100:00000001:3.0:1775776494.940936:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776494.940937:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287598336 : -114841421953280 : ffff978d65d6c700) 00000100:00000040:3.0:1775776494.940942:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.940946:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776494.940948:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776494.940949:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776494.940951:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099758080, offset 0 00000400:00000200:3.0:1775776494.940953:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776494.940956:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776494.940958:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776494.940960:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776494.940961:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776494.940963:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776494.940964:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776494.940966:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4153:4153:256:4294967295] 192.168.201.139@tcp LPNI seq info [4153:4153:8:4294967295] 00000400:00000200:3.0:1775776494.940969:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776494.940973:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776494.940976:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776494.940977:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776494.940980:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776494.940981:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776494.940984:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776494.940986:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776494.940988:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776494.940989:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776494.940991:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776494.940993:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776494.940995:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776494.940996:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776494.940997:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776494.940999:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941004:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.941005:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941009:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941010:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776494.941012:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776494.941014:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776494.941016:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776494.941017:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312882693 00000100:00000001:3.0:1775776494.941019:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312883957 00000100:00000001:3.0:1775776494.941020:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312882693 00000100:00000001:3.0:1775776494.941021:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312883957 00000100:00000001:3.0:1775776494.941022:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776494.941024:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941025:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776494.941026:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776494/real 1775776494] req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776494.941031:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776494.941032:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776494.941033:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776494.941034:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776494.941035:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776494.941036:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776494.941037:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776494.941042:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941045:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776494.941046:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941047:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.941048:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776494.941050:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941053:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776494.941055:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941059:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776494.941067:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941068:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.941069:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941070:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776494.941071:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776494.941073:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776494.941075:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776494.941076:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776494.941078:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776494.941083:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758080:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776494.941086:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776494.941089:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941095:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776494.941097:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.941098:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776494.941100:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776494.941102:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776494.941105:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758208:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776494.941107:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776494.941109:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776494.941110:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.941112:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776494.941115:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776494.941117:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.941121:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776494.941123:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776494.941125:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776494.941128:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099758208, portal 10 00000100:00000001:3.0:1775776494.941130:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776494.941131:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287597440 : -114841421954176 : ffff978d65d6c380) 00000100:00000040:3.0:1775776494.941136:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941141:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776494.941143:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776494.941145:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776494.941147:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099758208, offset 0 00000400:00000200:3.0:1775776494.941151:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776494.941154:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776494.941157:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776494.941160:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776494.941162:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776494.941165:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776494.941167:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776494.941169:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4154:4154:256:4294967295] 192.168.201.139@tcp LPNI seq info [4154:4154:8:4294967295] 00000400:00000200:3.0:1775776494.941174:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776494.941179:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776494.941182:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776494.941185:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776494.941187:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776494.941189:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776494.941191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776494.941193:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776494.941195:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776494.941196:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776494.941198:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776494.941200:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776494.941202:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776494.941203:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776494.941204:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776494.941207:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941211:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.941213:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941217:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941218:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776494.941220:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776494.941222:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776494.941224:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776494.941225:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312882694 00000100:00000001:3.0:1775776494.941226:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312883957 00000100:00000001:3.0:1775776494.941227:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312882694 00000100:00000001:3.0:1775776494.941229:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312883957 00000100:00000001:3.0:1775776494.941230:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776494.941231:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941232:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776494.941234:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776494/real 1775776494] req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776494.941238:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776494.941239:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776494.941240:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776494.941241:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776494.941242:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776494.941243:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776494.941245:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776494.941251:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941257:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776494.941258:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941260:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.941261:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776494.941264:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941271:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776494.941274:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941280:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776494.941282:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941284:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.941285:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941287:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776494.941288:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776494.941290:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776494.941292:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776494.941293:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776494.941295:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776494.941300:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758208:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776494.941302:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776494.941304:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941307:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776494.941308:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.941310:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776494.941311:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776494.941312:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776494.941314:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758336:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776494.941316:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776494.941317:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776494.941317:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.941318:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776494.941320:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776494.941321:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.941324:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776494.941325:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776494.941327:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776494.941329:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099758336, portal 4 00000100:00000001:3.0:1775776494.941330:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776494.941331:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287603712 : -114841421947904 : ffff978d65d6dc00) 00000100:00000040:3.0:1775776494.941335:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941338:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776494.941340:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776494.941341:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776494.941342:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099758336, offset 0 00000400:00000200:3.0:1775776494.941344:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776494.941346:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776494.941348:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776494.941350:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776494.941351:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776494.941353:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776494.941354:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776494.941356:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4155:4155:256:4294967295] 192.168.201.139@tcp LPNI seq info [4155:4155:8:4294967295] 00000400:00000200:3.0:1775776494.941359:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776494.941362:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776494.941365:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776494.941367:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776494.941369:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776494.941370:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776494.941373:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776494.941374:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776494.941376:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776494.941378:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776494.941379:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776494.941381:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776494.941383:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776494.941384:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776494.941385:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776494.941387:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941392:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.941394:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941397:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941398:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776494.941400:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776494.941401:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776494.941403:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776494.941404:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312882703 00000100:00000001:3.0:1775776494.941406:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312883957 00000100:00000001:3.0:1775776494.941407:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312882703 00000100:00000001:3.0:1775776494.941408:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312883957 00000100:00000001:3.0:1775776494.941409:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776494.941410:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941411:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776494.941413:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776494/real 1775776494] req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776494.941418:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776494.941419:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776494.941420:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776494.941422:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776494.941424:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776494.941425:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776494.941428:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776494.941435:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941441:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776494.941442:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941444:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.941445:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776494.941448:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941454:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776494.941457:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941464:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776494.941467:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941468:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.941469:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941471:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776494.941472:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776494.941475:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776494.941477:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776494.941479:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776494.941483:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776494.941490:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758336:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776494.941494:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776494.941497:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941503:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776494.941504:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.941506:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776494.941508:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776494.941510:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776494.941512:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758464:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776494.941515:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776494.941517:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776494.941518:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776494.941520:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776494.941523:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776494.941524:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776494.941527:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776494.941528:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776494.941530:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776494.941532:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099758464, portal 4 00000100:00000001:3.0:1775776494.941533:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776494.941534:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287611776 : -114841421939840 : ffff978d65d6fb80) 00000100:00000040:3.0:1775776494.941538:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941542:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776494.941543:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776494.941544:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776494.941546:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099758464, offset 0 00000400:00000200:3.0:1775776494.941548:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776494.941550:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776494.941552:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776494.941554:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776494.941556:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776494.941557:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776494.941559:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776494.941561:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4156:4156:256:4294967295] 192.168.201.139@tcp LPNI seq info [4156:4156:8:4294967295] 00000400:00000200:3.0:1775776494.941564:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776494.941570:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776494.941574:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776494.941577:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776494.941581:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776494.941583:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776494.941587:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776494.941590:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776494.941593:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776494.941595:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776494.941598:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776494.941601:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776494.941603:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776494.941605:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776494.941607:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776494.941610:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941615:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.941617:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776519 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941621:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941622:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776494.941625:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776494.941628:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776494.941631:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776494.941633:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312882694 00000100:00000001:3.0:1775776494.941635:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312883957 00000100:00000001:3.0:1775776494.941637:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312882694 00000100:00000001:3.0:1775776494.941639:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312883957 00000100:00000001:3.0:1775776494.941640:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776494.941642:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941644:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776494.941647:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776494/real 1775776494] req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776494.941655:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776494.941656:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776494.941658:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776494.941660:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776494.941661:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776494.941663:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776494.941666:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776494.941674:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941680:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776494.941681:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941683:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.941684:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776494.941687:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941694:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776494.941698:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941704:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776494.941707:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941709:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776494.941710:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941711:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776494.941713:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776494.941715:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776494.941718:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776494.941719:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776494.941721:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776494.941726:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758464:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776494.941728:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941729:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.941731:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c700 x1862033099758080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941734:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776494.941735:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776494.941737:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776494.941738:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776494.941739:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776494.941740:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776494.941741:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776494.941742:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3bc00. 02000000:00000001:3.0:1775776494.941744:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776494.941744:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776494.941745:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c700. 00000100:00000001:3.0:1775776494.941747:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776494.941747:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941749:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.941751:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c380 x1862033099758208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941762:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776494.941763:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776494.941764:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776494.941767:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776494.941767:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776494.941768:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776494.941770:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776494.941772:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39000. 02000000:00000001:3.0:1775776494.941774:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776494.941775:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776494.941776:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c380. 00000100:00000001:3.0:1775776494.941795:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776494.941796:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941798:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.941801:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6dc00 x1862033099758336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941805:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776494.941806:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776494.941807:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776494.941808:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776494.941809:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776494.941810:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776494.941811:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776494.941812:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3ac00. 02000000:00000001:3.0:1775776494.941813:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776494.941813:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776494.941814:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6dc00. 00000100:00000001:3.0:1775776494.941816:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776494.941816:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941817:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776494.941819:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6fb80 x1862033099758464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776519 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776494.941822:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776494.941823:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776494.941824:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776494.941825:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776494.941826:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776494.941826:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776494.941827:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776494.941828:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38000. 02000000:00000001:3.0:1775776494.941829:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776494.941830:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776494.941830:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6fb80. 00000100:00000001:3.0:1775776494.941831:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776494.941832:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941833:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776494.941835:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776494.941836:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776494.941837:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776494.941838:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776495.645878:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776495.645899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776495.645901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776495.645907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776495.645915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776495.645920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776495.645923:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776495.645926:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776495.645929:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776495.707882:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776495.707918:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776495.707921:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776495.707926:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776495.707932:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776495.707937:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776495.707941:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776495.707945:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776495.707949:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776495.707995:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776495.707999:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776495.708002:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776496.219869:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312883957, at 4312883968 00000100:00000040:0.0:1775776496.219874:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312883957, at 4312883968 00000100:00000010:0.0:1775776496.219904:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000100:00000010:1.0:1775776496.219906:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000100:00000001:3.0:1775776496.667945:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776496.667951:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776496.667953:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776496.667958:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776496.667961:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776496.667965:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776496.667967:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776496.667969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776496.667972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776496.731899:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776496.731908:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776496.731911:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776496.731918:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776496.731924:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776496.731929:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776496.731935:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776496.731938:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776496.731943:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776496.732000:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776496.732003:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776496.732005:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776497.691890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776497.691896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776497.691898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776497.691903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776497.691906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776497.691910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776497.691912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776497.691914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776497.691917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.755842:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776497.755848:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776497.755860:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776497.755864:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776497.755866:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776497.755869:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776497.755873:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776497.755876:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776497.755879:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776497.755952:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776497.755956:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776497.755959:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776498.715867:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776498.715874:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776498.715876:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776498.715881:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776498.715884:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776498.715888:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776498.715892:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776498.715894:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776498.715897:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776498.779887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776498.779892:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776498.779893:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776498.779898:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776498.779901:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776498.779903:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776498.779906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776498.779909:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776498.779912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776498.779963:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776498.779966:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776498.779968:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776499.739916:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776499.739922:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776499.739923:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776499.739927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776499.739929:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776499.739932:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776499.739934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776499.739936:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776499.739937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776499.803898:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776499.803904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776499.803906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776499.803911:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776499.803914:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776499.803922:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776499.803925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776499.803927:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776499.803930:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776499.804164:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776499.804166:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776499.804169:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776500.060122:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776500.060126:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776500.060129:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776500.060138:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060147:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776500.060150:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.060156:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776500.060158:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776500.060162:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776500.060168:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099758592:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776500.060172:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776500.060176:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776500.060180:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.060183:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776500.060187:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776500.060190:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776500.060198:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776500.060202:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776500.060211:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776500.060220:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099758592, portal 25 00000100:00000001:3.0:1775776500.060224:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776500.060227:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288710656 : -114841420840960 : ffff978d65e7c000) 00000100:00000040:3.0:1775776500.060239:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060262:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776500.060270:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776500.060275:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776500.060282:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099758592, offset 0 00000400:00000200:3.0:1775776500.060293:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776500.060302:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.060310:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776500.060317:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776500.060323:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776500.060330:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776500.060335:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776500.060340:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4157:4157:256:4294967295] 192.168.201.139@tcp LPNI seq info [4157:4157:8:4294967295] 00000400:00000200:3.0:1775776500.060348:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.060358:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776500.060369:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.060374:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776500.060385:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776500.060390:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776500.060401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776500.060408:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776500.060428:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.060431:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776500.060438:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776500.060445:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776500.060450:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776500.060453:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776500.060457:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776500.060463:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060476:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.060479:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060489:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.060492:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776500.060497:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776500.060504:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776500.060507:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312889077 00000100:00000010:3.0:1775776500.060511:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344c00. 00000100:00000001:3.0:1775776500.060518:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776500.060530:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312889077 00000100:00000010:3.0:1775776500.060535:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503440c0. 00000100:00000001:3.0:1775776500.060539:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776500.060545:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776500.060549:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.060552:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776500.060558:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776500/real 1775776500] req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776500.060574:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776500.060576:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776500.060579:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776500.060582:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776500.060584:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776500.060588:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776500.060593:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776500.060617:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060634:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776500.060637:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.060641:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.060645:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776500.060652:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060668:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776500.060677:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060692:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776500.060698:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.060702:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.060704:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.060710:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776500.060714:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776500.060720:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776500.060727:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776500.060730:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776500.060737:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776500.060754:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099758592:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776500.060760:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.060763:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.060767:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c000 x1862033099758592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060807:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776500.060822:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776500.060825:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776500.060830:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776500.060833:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776500.060836:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776500.060841:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776500.060844:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:3.0:1775776500.060850:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776500.060854:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776500.060857:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c000. 00000100:00000001:3.0:1775776500.060862:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776500.060865:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.060869:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.060876:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776500.060878:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.060881:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776500.060882:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776500.060884:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776500.060889:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060895:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776500.060897:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.060899:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776500.060901:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776500.060903:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776500.060905:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758720:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776500.060908:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776500.060910:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776500.060911:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.060912:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776500.060915:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776500.060921:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776500.060924:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776500.060927:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776500.060930:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776500.060933:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099758720, portal 10 00000100:00000001:3.0:1775776500.060934:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776500.060935:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288719616 : -114841420832000 : ffff978d65e7e300) 00000100:00000040:3.0:1775776500.060940:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.060945:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776500.060947:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776500.060949:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776500.060951:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099758720, offset 0 00000400:00000200:3.0:1775776500.060954:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776500.060957:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.060960:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776500.060963:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776500.060966:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776500.060968:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776500.060970:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776500.060973:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4158:4158:256:4294967295] 192.168.201.139@tcp LPNI seq info [4158:4158:8:4294967295] 00000400:00000200:3.0:1775776500.060976:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.060982:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776500.060986:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.060989:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776500.060993:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776500.060995:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776500.061000:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776500.061002:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776500.061005:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.061006:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776500.061009:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776500.061012:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776500.061015:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776500.061017:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776500.061019:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776500.061022:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061029:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.061031:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061038:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.061039:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776500.061042:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776500.061045:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776500.061048:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776500.061050:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312887813 00000100:00000001:3.0:1775776500.061052:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312889077 00000100:00000001:3.0:1775776500.061054:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312887813 00000100:00000001:3.0:1775776500.061056:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312889077 00000100:00000001:3.0:1775776500.061058:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776500.061060:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.061061:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776500.061065:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776500/real 1775776500] req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776500.061072:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776500.061073:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776500.061075:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776500.061077:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776500.061079:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776500.061081:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776500.061083:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776500.061091:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061096:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776500.061098:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061099:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.061100:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776500.061104:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061110:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776500.061113:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061119:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776500.061121:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061123:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.061124:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061125:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776500.061127:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776500.061130:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776500.061133:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776500.061135:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776500.061139:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776500.061146:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758720:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776500.061150:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776500.061153:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061159:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776500.061160:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.061162:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776500.061164:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776500.061165:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776500.061168:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758848:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776500.061171:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776500.061173:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776500.061174:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.061176:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776500.061178:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776500.061180:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776500.061183:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776500.061186:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776500.061188:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776500.061191:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099758848, portal 10 00000100:00000001:3.0:1775776500.061193:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776500.061194:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288725888 : -114841420825728 : ffff978d65e7fb80) 00000100:00000040:3.0:1775776500.061199:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061202:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776500.061204:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776500.061205:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776500.061207:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099758848, offset 0 00000400:00000200:3.0:1775776500.061210:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776500.061214:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.061217:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776500.061220:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776500.061222:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776500.061224:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776500.061226:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776500.061228:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4159:4159:256:4294967295] 192.168.201.139@tcp LPNI seq info [4159:4159:8:4294967295] 00000400:00000200:3.0:1775776500.061230:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.061234:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776500.061238:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.061241:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776500.061245:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776500.061247:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776500.061251:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776500.061252:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776500.061254:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.061256:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776500.061257:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776500.061260:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776500.061262:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776500.061263:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776500.061265:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776500.061268:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061276:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.061279:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061285:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.061286:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776500.061290:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776500.061293:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776500.061296:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776500.061298:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312887814 00000100:00000001:3.0:1775776500.061300:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312889077 00000100:00000001:3.0:1775776500.061302:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312887814 00000100:00000001:3.0:1775776500.061304:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312889077 00000100:00000001:3.0:1775776500.061306:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776500.061308:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.061309:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776500.061313:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776500/real 1775776500] req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776500.061321:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776500.061322:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776500.061324:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776500.061326:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776500.061328:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776500.061329:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776500.061332:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776500.061340:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061346:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776500.061347:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061349:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.061350:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776500.061353:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061360:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776500.061363:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061370:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776500.061372:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061373:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.061374:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061376:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776500.061377:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776500.061379:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776500.061381:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776500.061383:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776500.061386:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776500.061393:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758848:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776500.061397:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776500.061400:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061404:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776500.061405:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.061406:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776500.061407:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776500.061409:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776500.061410:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758976:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776500.061412:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776500.061414:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776500.061415:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.061417:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776500.061419:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776500.061421:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776500.061425:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776500.061426:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776500.061429:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776500.061431:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099758976, portal 4 00000100:00000001:3.0:1775776500.061432:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776500.061433:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288720512 : -114841420831104 : ffff978d65e7e680) 00000100:00000040:3.0:1775776500.061436:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061440:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776500.061443:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776500.061445:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776500.061447:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099758976, offset 0 00000400:00000200:3.0:1775776500.061451:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776500.061455:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.061458:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776500.061460:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776500.061463:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776500.061465:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776500.061467:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776500.061469:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4160:4160:256:4294967295] 192.168.201.139@tcp LPNI seq info [4160:4160:8:4294967295] 00000400:00000200:3.0:1775776500.061473:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.061478:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776500.061483:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.061485:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776500.061490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776500.061492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776500.061496:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776500.061498:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776500.061501:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.061503:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776500.061506:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776500.061509:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776500.061512:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776500.061513:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776500.061515:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776500.061519:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061525:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.061528:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061539:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.061541:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776500.061543:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776500.061546:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776500.061549:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776500.061551:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312887823 00000100:00000001:3.0:1775776500.061553:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312889077 00000100:00000001:3.0:1775776500.061555:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312887823 00000100:00000001:3.0:1775776500.061557:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312889077 00000100:00000001:3.0:1775776500.061559:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776500.061561:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.061563:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776500.061566:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776500/real 1775776500] req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776500.061572:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776500.061573:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776500.061574:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776500.061576:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776500.061578:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776500.061579:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776500.061582:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776500.061588:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061594:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776500.061595:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061597:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.061598:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776500.061601:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061607:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776500.061611:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061617:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776500.061619:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061621:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.061622:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061624:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776500.061625:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776500.061628:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776500.061631:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776500.061633:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776500.061637:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776500.061644:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099758976:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776500.061647:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776500.061650:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061653:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776500.061654:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.061656:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776500.061657:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776500.061658:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776500.061659:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759104:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776500.061661:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776500.061662:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776500.061664:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776500.061666:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776500.061668:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776500.061670:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776500.061674:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776500.061676:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776500.061678:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776500.061680:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099759104, portal 4 00000100:00000001:3.0:1775776500.061681:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776500.061682:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288721408 : -114841420830208 : ffff978d65e7ea00) 00000100:00000040:3.0:1775776500.061686:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061690:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776500.061691:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776500.061692:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776500.061694:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099759104, offset 0 00000400:00000200:3.0:1775776500.061698:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776500.061701:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.061705:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776500.061707:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776500.061708:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776500.061710:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776500.061711:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776500.061713:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4161:4161:256:4294967295] 192.168.201.139@tcp LPNI seq info [4161:4161:8:4294967295] 00000400:00000200:3.0:1775776500.061715:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776500.061719:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776500.061723:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.061726:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776500.061730:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776500.061732:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776500.061735:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776500.061737:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776500.061739:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776500.061741:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776500.061743:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776500.061745:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776500.061746:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776500.061747:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776500.061749:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776500.061752:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061759:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.061762:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776525 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061765:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.061766:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776500.061768:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776500.061770:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776500.061772:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776500.061773:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312887814 00000100:00000001:3.0:1775776500.061775:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312889077 00000100:00000001:3.0:1775776500.061776:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312887814 00000100:00000001:3.0:1775776500.061795:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312889077 00000100:00000001:3.0:1775776500.061798:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776500.061799:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.061801:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776500.061805:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776500/real 1775776500] req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776500.061825:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776500.061826:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776500.061828:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776500.061831:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776500.061832:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776500.061835:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776500.061838:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776500.061846:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061852:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776500.061854:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061855:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.061856:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776500.061859:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061865:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776500.061869:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061875:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776500.061877:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061879:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776500.061880:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061881:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776500.061882:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776500.061885:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776500.061896:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776500.061899:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776500.061906:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776500.061914:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759104:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776500.061917:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061919:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.061922:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e300 x1862033099758720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061929:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776500.061930:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776500.061932:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776500.061934:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776500.061936:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776500.061937:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776500.061939:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776500.061940:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:3.0:1775776500.061943:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776500.061944:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776500.061945:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e300. 00000100:00000001:3.0:1775776500.061947:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776500.061949:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061950:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.061953:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7fb80 x1862033099758848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061957:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776500.061958:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776500.061959:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776500.061961:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776500.061962:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776500.061963:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776500.061965:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776500.061966:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:3.0:1775776500.061969:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776500.061970:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776500.061971:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7fb80. 00000100:00000001:3.0:1775776500.061972:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776500.061973:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061974:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.061976:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e680 x1862033099758976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.061979:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776500.061980:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776500.061981:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776500.061982:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776500.061983:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776500.061984:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776500.061985:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776500.061987:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:3.0:1775776500.061989:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776500.061989:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776500.061991:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e680. 00000100:00000001:3.0:1775776500.061993:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776500.061994:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.061995:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776500.061998:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ea00 x1862033099759104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776525 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776500.062004:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776500.062005:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776500.062006:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776500.062008:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776500.062009:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776500.062010:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776500.062012:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776500.062013:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505c00. 02000000:00000001:3.0:1775776500.062015:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776500.062016:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776500.062017:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ea00. 00000100:00000001:3.0:1775776500.062019:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776500.062020:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.062022:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.062025:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776500.062026:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.062028:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776500.062030:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.763883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776500.763890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776500.763892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776500.763897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776500.763901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776500.763905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.763908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776500.763910:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.763912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776500.827911:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776500.827919:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776500.827923:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776500.827929:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776500.827935:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776500.827939:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.827945:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776500.827949:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776500.827954:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776500.828024:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776500.828027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776500.828030:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776501.339889:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312889077, at 4312889088 00000100:00000040:1.0:1775776501.339889:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312889077, at 4312889088 00000100:00000010:1.0:1775776501.339904:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503440c0. 00000100:00000010:0.0:1775776501.339907:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000100:00000001:3.0:1775776501.787887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776501.787892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776501.787894:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776501.787897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776501.787900:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776501.787902:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776501.787904:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776501.787906:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776501.787908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776501.851894:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776501.851901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776501.851903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776501.851907:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776501.851911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776501.851915:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776501.851918:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776501.851920:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776501.851924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776501.851981:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776501.851984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776501.851987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776502.811865:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776502.811871:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776502.811872:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776502.811876:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776502.811879:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776502.811881:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776502.811884:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776502.811886:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776502.811887:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776502.875966:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776502.875969:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776502.875971:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776502.876013:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776502.876017:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776502.876019:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776502.876023:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776502.876027:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776502.876030:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776502.876032:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776502.876034:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776502.876037:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776503.835884:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776503.835891:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776503.835893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776503.835898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776503.835901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776503.835905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776503.835907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776503.835908:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776503.835910:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776503.899872:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776503.899881:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776503.899884:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776503.899890:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776503.899897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776503.899901:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776503.899904:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776503.899906:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776503.899909:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776503.899948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776503.899952:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776503.899955:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776504.859843:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776504.859849:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776504.859851:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776504.859854:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776504.859857:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776504.859860:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776504.859862:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776504.859864:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776504.859867:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776504.923845:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776504.923851:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776504.923853:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776504.923859:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776504.923863:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776504.923866:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776504.923869:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776504.923872:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776504.923875:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776504.923922:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776504.923925:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776504.923928:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776505.179888:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776505.179896:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776505.179898:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776505.179903:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776505.179906:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776505.179911:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18220 00000400:00000200:1.0:1775776505.179926:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776505.179931:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776505.179934:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776505.179936:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.179939:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776505.179941:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.179942:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776505.179945:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776505.179946:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776505.179948:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776505.179950:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776505.179952:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776505.179954:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776505.179956:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.179957:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776505.179959:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.179960:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776505.179962:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776505.179964:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776505.179965:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776505.179967:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776505.179970:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776505.179972:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.179973:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776505.179975:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776505.179983:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776505.179986:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776505.179989:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776505.179990:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.179993:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776505.179995:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776505.179998:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776505.180006:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776505.180011:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776505.180018:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776505.180028:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776505.180033:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776505.180038:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776505.180066:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776505.180069:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776505.180073:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776505.180074:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776505.180076:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776505.180079:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776505.180082:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18220 00000400:00000200:1.0:1775776505.180095:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776505.180099:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776505.180102:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776505.180106:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.180109:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776505.180109:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1775776505.180114:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776505.180115:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776505.180116:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776505.180119:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1775776505.180120:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776505.180122:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776505.180125:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776505.180126:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776505.180129:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776505.180130:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775776505.180131:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776505.180138:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.180142:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776505.180146:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.180151:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776505.180153:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1775776505.180154:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000001:3.0:1775776505.180156:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776505.180157:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 02000000:00000001:3.0:1775776505.180160:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775776505.180161:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776505.180164:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776505.180164:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776505.180167:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776505.180169:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1775776505.180171:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099759232:192.168.201.139@tcp:250:kworker.0 02000000:00000001:1.0:1775776505.180172:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180177:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776505.180178:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:1.0:1775776505.180181:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1775776505.180183:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:1.0:1775776505.180184:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.180186:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180187:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.180190:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:1.0:1775776505.180192:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776505.180194:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000010:3.0:1775776505.180195:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:1.0:1775776505.180197:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776505.180199:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.180199:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180204:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776505.180206:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1775776505.180210:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:1.0:1775776505.180215:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776505.180215:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 02000000:00000010:1.0:1775776505.180221:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776505.180225:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776505.180226:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00100000:1.0:1775776505.180231:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1775776505.180235:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099759232, portal 25 00000100:00000001:1.0:1775776505.180239:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776505.180241:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776505.180242:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.180245:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287611776 : -114841421939840 : ffff978d65d6fb80) 00000100:00000040:1.0:1775776505.180247:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776505.180256:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776505.180257:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776505.180261:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776505.180269:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776505.180271:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776505.180273:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776505.180279:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776505.180284:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18220 00000100:00000001:3.0:1775776505.180286:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1775776505.180289:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775776505.180293:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000200:1.0:1775776505.180296:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000010:3.0:1775776505.180296:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00080000:1.0:1775776505.180300:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776505.180303:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000200:3.0:1775776505.180303:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099759232, offset 0 00000100:00000001:1.0:1775776505.180308:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776505.180310:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.180314:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:3.0:1775776505.180315:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000020:00000001:1.0:1775776505.180318:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776505.180320:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776505.180321:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776505.180323:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776505.180325:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:3.0:1775776505.180325:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:1.0:1775776505.180329:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776505.180334:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.180335:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:1.0:1775776505.180337:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776505.180340:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.180341:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775776505.180344:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776505.180348:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:3.0:1775776505.180348:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00080000:1.0:1775776505.180353:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000200:3.0:1775776505.180354:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775776505.180357:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776505.180359:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 02000000:00000001:1.0:1775776505.180360:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776505.180363:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775776505.180367:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4162:4162:256:4294967295] 192.168.201.139@tcp LPNI seq info [4162:4162:8:4294967295] 02000000:00000001:1.0:1775776505.180368:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180372:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776505.180376:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1775776505.180378:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000008:00000001:1.0:1775776505.180380:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180382:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776505.180385:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776505.180390:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000400:00000200:3.0:1775776505.180391:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 02000000:00000001:1.0:1775776505.180394:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776505.180397:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180399:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:3.0:1775776505.180400:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 02000000:00000001:1.0:1775776505.180402:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776505.180404:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775776505.180405:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 02000000:00000010:1.0:1775776505.180408:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776505.180411:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775776505.180413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776505.180417:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00100000:1.0:1775776505.180424:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1775776505.180429:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776505.180436:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:3.1:1775776505.180436:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1775776505.180438:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776505.180444:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1775776505.180449:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.180454:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000100:00000001:1.0:1775776505.180456:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776505.180459:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776505.180461:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776505.180462:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1775776505.180463:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1775776505.180464:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776505.180469:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776505.180474:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18220 00000400:00000200:1.0:1775776505.180478:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776505.180480:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776505.180482:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776505.180485:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.180485:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775776505.180489:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776505.180491:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.180493:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000100:00000001:1.0:1775776505.180494:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776505.180497:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:3.0:1775776505.180497:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000020:00000040:1.0:1775776505.180500:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776505.180502:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776505.180503:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:1.0:1775776505.180504:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776505.180507:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776505.180510:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000200:3.0:1775776505.180511:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776505.180515:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.180519:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776505.180523:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.180527:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776505.180530:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000001:3.0:1775776505.180531:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1775776505.180534:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776505.180538:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776505.180538:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776505.180542:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776505.180545:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776505.180549:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180554:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776505.180556:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776505.180561:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.180563:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180564:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.180569:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:1.0:1775776505.180570:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776505.180572:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00000001:3.0:1775776505.180574:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776505.180575:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776505.180577:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180579:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776505.180581:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1775776505.180582:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:1.0:1775776505.180583:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.180585:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312894197 02000000:00000010:1.0:1775776505.180587:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776505.180590:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776505.180591:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344ba0. 00000100:00100000:1.0:1775776505.180595:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776505.180598:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776505.180608:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776505.180610:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776505.180612:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.180614:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312894197 00000100:00000010:3.0:1775776505.180620:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344d20. 00000100:00000001:1.0:1775776505.180625:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.180625:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776505.180629:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776505.180631:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776505.180633:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775776505.180634:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1775776505.180638:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776505.180639:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776505.180642:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775776505.180642:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776505/real 1775776505] req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776505.180647:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776505.180652:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18220 00000400:00000200:1.0:1775776505.180658:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000001:3.0:1775776505.180659:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:1.0:1775776505.180662:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000001:3.0:1775776505.180662:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1775776505.180666:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000010:3.0:1775776505.180666:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000100:00000040:1.0:1775776505.180671:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.180671:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000100:00000001:1.0:1775776505.180675:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1775776505.180675:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000040:1.0:1775776505.180679:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776505.180680:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1775776505.180681:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776505.180684:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000200:3.0:1775776505.180685:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775776505.180689:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776505.180692:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776505.180694:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776505.180697:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000200:3.0:1775776505.180697:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775776505.180701:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776505.180709:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776505.180712:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776505.180714:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:1.0:1775776505.180717:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776505.180717:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776505.180738:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776505.180742:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776505.180746:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776505.180747:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.180750:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776505.180751:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776505.180755:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1775776505.180755:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776505.180759:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776505.180763:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776505.180767:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776505.180772:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180777:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776505.180806:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776505.180809:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776505.180811:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776505.180816:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776505.180820:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:3.0:1775776505.180823:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1775776505.180825:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776505.180828:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.180829:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776505.180832:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775776505.180833:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1775776505.180836:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1775776505.180836:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.180840:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1775776505.180841:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.180842:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776505.180846:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000010:1.0:1775776505.180849:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00080000:3.0:1775776505.180851:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776505.180854:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1775776505.180856:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776505.180858:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776505.180866:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775776505.180869:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099759232:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776505.180877:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.180881:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775776505.180884:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1775776505.180885:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6fb80 x1862033099759232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776505.180889:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776505.180897:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.180897:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776505.180901:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776505.180904:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 00000100:00000001:1.0:1775776505.180907:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.180908:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776505.180911:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:1.0:1775776505.180912:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18230) 00000020:00000040:3.0:1775776505.180912:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776505.180916:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776505.180918:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b000. 02000000:00000001:3.0:1775776505.180922:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776505.180925:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776505.180927:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6fb80. 00000100:00000001:3.0:1775776505.180930:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776505.180932:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.180935:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.180943:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776505.180945:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.180947:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776505.180949:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776505.180950:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776505.180954:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.180963:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776505.180964:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.180967:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776505.180969:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776505.180971:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776505.180974:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759360:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776505.180978:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776505.180980:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776505.180981:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.180982:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776505.180985:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776505.180986:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776505.180990:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776505.180991:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776505.180994:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776505.180996:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099759360, portal 10 00000100:00000001:3.0:1775776505.180998:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776505.180999:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287603712 : -114841421947904 : ffff978d65d6dc00) 00000100:00000040:3.0:1775776505.181004:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181008:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776505.181010:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776505.181011:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776505.181013:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099759360, offset 0 00000400:00000200:3.0:1775776505.181016:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776505.181019:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776505.181022:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776505.181024:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776505.181026:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776505.181027:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776505.181029:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.181031:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4163:4163:256:4294967295] 192.168.201.139@tcp LPNI seq info [4163:4163:8:4294967295] 00000400:00000200:3.0:1775776505.181034:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776505.181038:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776505.181041:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.181043:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776505.181046:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776505.181048:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776505.181051:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776505.181053:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1775776505.181055:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.181057:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776505.181059:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776505.181061:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776505.181063:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776505.181065:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776505.181066:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776505.181069:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181074:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776505.181076:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181082:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181084:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776505.181087:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776505.181091:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776505.181094:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776505.181096:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312892933 00000100:00000001:3.0:1775776505.181099:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312894197 00000100:00000001:3.0:1775776505.181101:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312892933 00000100:00000001:3.0:1775776505.181103:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312894197 00000100:00000001:3.0:1775776505.181105:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776505.181107:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181109:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776505.181113:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776505/real 1775776505] req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776505.181123:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776505.181124:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776505.181126:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776505.181129:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776505.181130:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776505.181133:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776505.181136:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776505.181145:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181151:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776505.181152:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181153:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.181154:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776505.181156:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181161:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776505.181163:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181168:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776505.181170:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181171:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.181172:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181173:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776505.181175:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776505.181176:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776505.181178:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776505.181180:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776505.181182:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776505.181187:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759360:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776505.181190:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776505.181192:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181197:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776505.181198:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.181200:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776505.181202:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776505.181204:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776505.181207:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759488:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776505.181211:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776505.181213:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776505.181214:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.181216:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776505.181220:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776505.181222:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776505.181226:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776505.181229:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776505.181233:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776505.181236:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099759488, portal 10 00000100:00000001:3.0:1775776505.181238:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776505.181239:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287597440 : -114841421954176 : ffff978d65d6c380) 00000100:00000040:3.0:1775776505.181246:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181254:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776505.181256:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776505.181258:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776505.181260:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099759488, offset 0 00000400:00000200:3.0:1775776505.181263:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776505.181266:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776505.181269:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776505.181271:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776505.181273:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776505.181275:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776505.181276:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.181279:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4164:4164:256:4294967295] 192.168.201.139@tcp LPNI seq info [4164:4164:8:4294967295] 00000400:00000200:3.0:1775776505.181281:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776505.181286:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776505.181288:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.181290:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776505.181293:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776505.181295:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776505.181297:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776505.181300:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1775776505.181302:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.181304:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776505.181306:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776505.181309:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776505.181311:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776505.181312:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776505.181314:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776505.181317:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181323:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776505.181325:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181329:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181330:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776505.181332:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776505.181334:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776505.181336:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776505.181338:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312892934 00000100:00000001:3.0:1775776505.181340:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312894197 00000100:00000001:3.0:1775776505.181341:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312892934 00000100:00000001:3.0:1775776505.181343:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312894197 00000100:00000001:3.0:1775776505.181344:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776505.181346:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181347:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776505.181349:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776505/real 1775776505] req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776505.181353:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776505.181354:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776505.181355:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776505.181357:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776505.181358:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776505.181359:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776505.181361:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776505.181366:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181371:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776505.181372:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181373:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.181374:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776505.181376:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181380:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776505.181382:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181386:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776505.181388:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181389:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.181390:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181391:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776505.181392:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776505.181393:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776505.181395:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776505.181396:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776505.181399:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776505.181403:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759488:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776505.181406:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776505.181407:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181412:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776505.181413:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.181416:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776505.181417:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776505.181419:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776505.181422:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759616:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776505.181426:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776505.181427:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776505.181429:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.181431:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776505.181434:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776505.181436:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776505.181439:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776505.181442:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776505.181445:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776505.181448:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099759616, portal 4 00000100:00000001:3.0:1775776505.181450:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776505.181452:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287598336 : -114841421953280 : ffff978d65d6c700) 00000100:00000040:3.0:1775776505.181458:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181465:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776505.181467:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776505.181469:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776505.181472:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099759616, offset 0 00000400:00000200:3.0:1775776505.181477:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776505.181481:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776505.181483:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776505.181485:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776505.181487:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776505.181489:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776505.181490:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.181492:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4165:4165:256:4294967295] 192.168.201.139@tcp LPNI seq info [4165:4165:8:4294967295] 00000400:00000200:3.0:1775776505.181496:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776505.181500:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776505.181503:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.181506:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776505.181508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776505.181510:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776505.181513:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776505.181514:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1775776505.181517:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.181518:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776505.181520:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776505.181523:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776505.181525:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776505.181526:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776505.181527:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776505.181530:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181535:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776505.181537:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181541:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181542:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776505.181544:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776505.181546:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776505.181548:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776505.181550:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312892943 00000100:00000001:3.0:1775776505.181551:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312894197 00000100:00000001:3.0:1775776505.181553:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312892943 00000100:00000001:3.0:1775776505.181554:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312894197 00000100:00000001:3.0:1775776505.181556:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776505.181557:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181558:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776505.181560:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776505/real 1775776505] req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776505.181566:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776505.181566:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776505.181568:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776505.181569:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776505.181570:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776505.181571:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776505.181573:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776505.181578:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181582:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776505.181583:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181584:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.181585:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776505.181587:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181591:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776505.181593:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181597:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776505.181598:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181599:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.181600:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181601:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776505.181602:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776505.181603:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776505.181605:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776505.181606:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776505.181609:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776505.181614:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759616:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776505.181617:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776505.181619:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181622:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776505.181623:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.181624:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776505.181625:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776505.181627:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776505.181628:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759744:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776505.181630:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776505.181631:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776505.181632:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776505.181633:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776505.181635:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776505.181636:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776505.181638:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776505.181640:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000400:00000010:3.0:1775776505.181642:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776505.181643:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099759744, portal 4 00000100:00000001:3.0:1775776505.181645:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776505.181646:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609088 : -114841421942528 : ffff978d65d6f100) 00000100:00000040:3.0:1775776505.181649:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181653:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776505.181655:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776505.181656:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000100:00000200:3.0:1775776505.181658:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099759744, offset 0 00000400:00000200:3.0:1775776505.181661:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776505.181663:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776505.181665:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776505.181667:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776505.181668:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776505.181670:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776505.181671:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776505.181673:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4166:4166:256:4294967295] 192.168.201.139@tcp LPNI seq info [4166:4166:8:4294967295] 00000400:00000200:3.0:1775776505.181676:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776505.181679:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776505.181682:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.181684:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776505.181686:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776505.181687:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775776505.181690:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776505.181691:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1775776505.181694:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776505.181695:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:3.0:1775776505.181697:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776505.181699:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776505.181701:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:3.0:1775776505.181702:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:3.0:1775776505.181703:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776505.181705:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181709:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776505.181711:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181715:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181716:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776505.181717:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776505.181734:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776505.181737:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776505.181739:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312892934 00000100:00000001:3.0:1775776505.181741:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312894197 00000100:00000001:3.0:1775776505.181742:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312892934 00000100:00000001:3.0:1775776505.181744:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312894197 00000100:00000001:3.0:1775776505.181745:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776505.181746:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181747:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776505.181750:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776505/real 1775776505] req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776505.181755:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776505.181756:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776505.181757:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776505.181759:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776505.181760:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776505.181761:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776505.181763:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776505.181768:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181775:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776505.181794:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181797:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.181798:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776505.181803:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181811:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776505.181815:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181823:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776505.181826:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181828:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776505.181829:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181831:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776505.181833:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776505.181835:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776505.181837:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776505.181839:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776505.181841:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776505.181846:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099759744:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776505.181849:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181850:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776505.181852:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6dc00 x1862033099759360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181856:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776505.181857:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776505.181859:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776505.181860:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776505.181861:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776505.181862:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776505.181864:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776505.181865:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39c00. 02000000:00000001:3.0:1775776505.181867:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776505.181868:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776505.181869:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6dc00. 00000100:00000001:3.0:1775776505.181871:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776505.181871:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181872:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776505.181874:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c380 x1862033099759488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181882:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776505.181883:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776505.181884:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776505.181886:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776505.181887:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776505.181887:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776505.181888:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776505.181889:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a000. 02000000:00000001:3.0:1775776505.181890:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776505.181891:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776505.181892:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c380. 00000100:00000001:3.0:1775776505.181893:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776505.181894:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181895:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776505.181896:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c700 x1862033099759616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181900:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776505.181901:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776505.181902:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776505.181903:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776505.181903:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776505.181904:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776505.181905:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776505.181906:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38000. 02000000:00000001:3.0:1775776505.181907:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776505.181908:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776505.181908:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c700. 00000100:00000001:3.0:1775776505.181909:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776505.181910:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181911:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776505.181913:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f100 x1862033099759744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776530 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776505.181916:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776505.181917:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776505.181918:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776505.181919:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776505.181920:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776505.181921:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776505.181923:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776505.181924:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3ac00. 02000000:00000001:3.0:1775776505.181926:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776505.181927:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776505.181929:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f100. 00000100:00000001:3.0:1775776505.181931:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776505.181932:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181934:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.181938:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776505.181939:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.181941:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776505.181943:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.883911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776505.883919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776505.883921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776505.883927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776505.883930:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776505.883935:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.883938:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776505.883940:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.883944:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776505.947885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776505.947891:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776505.947895:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776505.947902:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776505.947907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776505.947911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.947916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776505.947919:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776505.947923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776505.947985:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776505.947989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776505.947993:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776506.459853:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312894197, at 4312894208 00000100:00000010:0.0:1775776506.459869:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000100:00000040:1.0:1775776506.459873:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312894197, at 4312894208 00000100:00000010:1.0:1775776506.459882:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344d20. 00000100:00000001:3.0:1775776506.907845:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776506.907851:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776506.907852:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776506.907857:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776506.907860:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776506.907863:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776506.907865:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776506.907867:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776506.907876:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776506.971850:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776506.971858:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776506.971860:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776506.971865:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776506.971869:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776506.971872:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776506.971875:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776506.971878:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776506.971881:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776506.972042:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776506.972046:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776506.972049:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776507.931887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776507.931895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776507.931897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776507.931903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776507.931907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776507.931912:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776507.931914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776507.931916:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776507.931918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776507.995893:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776507.995899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776507.995901:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776507.995905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776507.995910:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776507.995914:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776507.995918:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776507.995920:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776507.995925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776507.995979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776507.995983:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776507.995986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776508.955891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776508.955896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776508.955898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776508.955903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776508.955906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776508.955909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776508.955911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776508.955913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776508.955915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776509.019882:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776509.019889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776509.019893:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776509.019900:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776509.019906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776509.019911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776509.019916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776509.019920:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776509.019924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776509.019978:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776509.019981:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776509.019983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776509.979849:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776509.979854:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776509.979856:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776509.979861:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776509.979863:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776509.979866:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776509.979868:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776509.979870:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776509.979872:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.043886:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776510.043893:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776510.043895:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776510.043901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776510.043905:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776510.043909:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776510.043912:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776510.043916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776510.043919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776510.044034:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776510.044038:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776510.044042:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776510.299893:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776510.299904:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776510.299906:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776510.299909:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776510.299912:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776510.299919:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18225 00000400:00000200:0.0:1775776510.299927:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776510.299932:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776510.299935:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776510.299937:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.299939:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776510.299942:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.299943:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776510.299946:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776510.299948:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776510.299950:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776510.299952:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776510.299953:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776510.299955:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776510.299956:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.299958:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776510.299959:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.299960:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776510.299963:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776510.299965:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776510.299967:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776510.299969:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776510.299972:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776510.299974:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.299976:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776510.299978:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776510.299985:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776510.299988:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776510.299991:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776510.299993:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.299995:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776510.299997:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776510.300001:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776510.300007:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776510.300013:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776510.300020:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776510.300028:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776510.300031:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776510.300035:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776510.300053:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776510.300055:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776510.300058:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776510.300059:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776510.300060:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776510.300062:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776510.300064:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18225 00000400:00000200:0.0:1775776510.300067:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776510.300070:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776510.300071:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776510.300074:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300076:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776510.300077:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776510.300078:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776510.300079:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776510.300083:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775776510.300083:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1775776510.300084:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776510.300087:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776510.300088:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1775776510.300090:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776510.300093:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776510.300095:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776510.300096:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776510.300100:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300105:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776510.300108:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300112:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776510.300117:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000001:3.0:1775776510.300118:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775776510.300120:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776510.300122:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300123:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776510.300128:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776510.300130:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776510.300133:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776510.300134:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776510.300136:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300139:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:3.0:1775776510.300140:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776510.300143:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776510.300147:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776510.300148:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099759872:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776510.300149:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776510.300155:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000100:00000001:3.0:1775776510.300156:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1775776510.300158:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776510.300161:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776510.300162:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.300162:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776510.300166:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776510.300167:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776510.300167:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776510.300169:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.300172:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776510.300174:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776510.300178:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776510.300179:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 00000100:00100000:0.0:1775776510.300184:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1775776510.300184:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300192:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776510.300196:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.300196:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:0.0:1775776510.300202:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775776510.300202:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d6e0. 00000100:00000001:0.0:1775776510.300211:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776510.300211:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00080000:0.0:1775776510.300216:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776510.300219:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776510.300220:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:3.0:1775776510.300220:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099759872, portal 25 00000100:00080000:0.0:1775776510.300223:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776510.300224:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776510.300226:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776510.300227:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00080000:0.0:1775776510.300230:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18225 00000400:00000200:0.0:1775776510.300233:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775776510.300236:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776510.300239:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776510.300243:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776510.300249:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300253:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776510.300254:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776510.300256:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300259:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000040:3.0:1775776510.300260:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000020:00000001:0.0:1775776510.300262:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:3.0:1775776510.300263:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1ee0. 00000020:00000040:0.0:1775776510.300264:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776510.300267:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:3.0:1775776510.300268:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099759872, offset 0 00000020:00000040:0.0:1775776510.300269:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776510.300272:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776510.300275:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:3.0:1775776510.300279:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000040:0.0:1775776510.300281:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300284:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776510.300285:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300286:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776510.300290:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:3.0:1775776510.300290:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:0.0:1775776510.300294:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000200:3.0:1775776510.300298:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:0.0:1775776510.300299:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776510.300302:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776510.300304:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775776510.300304:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1775776510.300308:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300311:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:3.0:1775776510.300311:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000008:00100000:0.0:1775776510.300314:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776510.300318:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.300318:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775776510.300322:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.300324:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000010:0.0:1775776510.300326:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776510.300330:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000400:00000200:3.0:1775776510.300331:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4167:4167:256:4294967295] 192.168.201.139@tcp LPNI seq info [4167:4167:8:4294967295] 02000000:00000001:0.0:1775776510.300334:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776510.300337:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300341:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1775776510.300341:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:0.0:1775776510.300344:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776510.300346:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776510.300349:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776510.300354:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.300357:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00100000:0.0:1775776510.300362:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1775776510.300371:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300373:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776510.300377:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776510.300381:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1775776510.300381:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511e00. 00000800:00000200:3.0:1775776510.300393:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776510.300401:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775776510.300402:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:0.0:1775776510.300404:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776510.300409:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776510.300410:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776510.300411:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776510.300413:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776510.300415:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18225 00000400:00000200:0.0:1775776510.300418:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776510.300421:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776510.300422:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776510.300424:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300425:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776510.300427:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300429:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776510.300430:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775776510.300430:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776510.300432:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776510.300435:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776510.300436:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776510.300441:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000400:00000200:3.0:1775776510.300441:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:0.0:1775776510.300445:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:3.0:1775776510.300446:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312899317 00000100:00000040:0.0:1775776510.300451:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300453:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:3.0:1775776510.300454:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503449c0. 00000100:00000040:0.0:1775776510.300456:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300459:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776510.300461:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000001:3.0:1775776510.300462:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1775776510.300464:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300467:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776510.300469:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776510.300471:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776510.300471:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312899317 02000000:00000001:0.0:1775776510.300476:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300478:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000010:3.0:1775776510.300478:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503446c0. 00000008:00100000:0.0:1775776510.300480:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1775776510.300482:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1775776510.300483:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300486:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.300488:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1775776510.300490:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000100:00000001:3.0:1775776510.300491:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776510.300494:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00000001:3.0:1775776510.300495:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776510.300498:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775776510.300499:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775776510.300501:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300503:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776510.300505:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776510.300506:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.300506:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776510.300509:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1775776510.300512:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1775776510.300513:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:3.0:1775776510.300515:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776510.300517:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.300518:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1775776510.300522:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1775776510.300525:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776510.300543:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776510.300544:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776510.300547:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.300548:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.300553:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775776510.300556:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776510.300556:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776510.300561:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776510.300568:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760000:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776510.300571:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776510.300572:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.300574:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1775776510.300575:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1775776510.300577:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776510.300578:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776510.300580:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1775776510.300580:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1775776510.300583:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776510.300585:0:303560:0:(import.c:484:import_select_connection()) Process entered 02000000:00000010:3.0:1775776510.300585:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776510.300589:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776510.300590:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18225 00000400:00000200:0.0:1775776510.300595:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776510.300598:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776510.300601:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000010:3.0:1775776510.300602:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d580. 00000100:00080000:0.0:1775776510.300604:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000010:3.0:1775776510.300607:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1110. 00000100:00000040:0.0:1775776510.300608:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300612:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1775776510.300613:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099760000, portal 10 00000100:00000040:0.0:1775776510.300615:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776510.300617:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776510.300618:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776510.300619:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741762432 : -114841967789184 : ffff978d454dfb80) 00000020:00000001:0.0:1775776510.300622:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776510.300624:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776510.300626:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776510.300629:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000100:00000040:3.0:1775776510.300629:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776510.300648:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1775776510.300649:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776510.300654:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776510.300654:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776510.300658:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1e58. 00000100:00000040:0.0:1775776510.300661:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000200:3.0:1775776510.300663:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099760000, offset 0 00000100:00000001:0.0:1775776510.300664:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776510.300668:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300672:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:3.0:1775776510.300672:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000020:00000040:0.0:1775776510.300676:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000400:00000200:3.0:1775776510.300680:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:0.0:1775776510.300682:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776510.300685:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.300687:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 02000000:00000001:0.0:1775776510.300688:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776510.300692:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775776510.300692:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1775776510.300696:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.300696:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000001:0.0:1775776510.300699:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:3.0:1775776510.300701:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000008:00100000:0.0:1775776510.300703:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000400:00000200:3.0:1775776510.300705:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000008:00000001:0.0:1775776510.300707:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300711:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.300713:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4168:4168:255:4294967295] 192.168.201.139@tcp LPNI seq info [4168:4168:7:4294967295] 00000100:00000010:0.0:1775776510.300715:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776510.300720:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000400:00000200:3.0:1775776510.300725:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:0.0:1775776510.300726:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776510.300730:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776510.300733:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1775776510.300739:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 02000000:00000001:0.0:1775776510.300741:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776510.300745:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775776510.300746:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 02000000:00000010:0.0:1775776510.300751:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000800:00000010:3.0:1775776510.300753:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511d00. 00000100:00000001:0.0:1775776510.300759:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775776510.300763:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00100000:0.0:1775776510.300769:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1775776510.300773:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776510.300815:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776510.300820:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776510.300825:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776510.300830:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312898053 00000100:00000001:0.0:1775776510.300833:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776510.300839:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.300839:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312899317 00000100:00000001:3.0:1775776510.300845:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312898053 00000100:00000040:0.0:1775776510.300849:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776510.300850:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312899317 00000100:00000001:3.0:1775776510.300856:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776510.300862:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.300868:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776510.300869:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776510.300876:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18235) 00000100:00000040:3.0:1775776510.300879:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776510.300888:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776510.300891:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.300893:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776510.300896:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776510.300898:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776510.300901:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760128:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776510.300905:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776510.300907:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776510.300909:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.300911:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776510.300915:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776510.300918:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.300921:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776510.300924:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063dec8. 00000400:00000010:3.0:1775776510.300928:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1990. 00000100:00000200:3.0:1775776510.300932:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099760128, portal 10 00000100:00000001:3.0:1775776510.300935:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776510.300936:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757056 : -114841967794560 : ffff978d454de680) 00000100:00000040:3.0:1775776510.300943:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776510.300951:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776510.300954:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776510.300957:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1330. 00000100:00000200:3.0:1775776510.300961:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099760128, offset 0 00000400:00000200:3.0:1775776510.300966:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776510.300972:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776510.300977:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776510.300980:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776510.300983:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776510.300985:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776510.300987:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776510.300990:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4169:4169:254:4294967295] 192.168.201.139@tcp LPNI seq info [4169:4169:6:4294967295] 00000400:00000200:3.0:1775776510.300994:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776510.301001:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776510.301005:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776510.301009:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511200. 00000800:00000200:3.0:1775776510.301013:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776510.301018:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776510.301020:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776510.301024:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776510.301027:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776510.301029:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312898054 00000100:00000001:3.0:1775776510.301031:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312899317 00000100:00000001:3.0:1775776510.301034:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312898054 00000100:00000001:3.0:1775776510.301036:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312899317 00000100:00000001:3.0:1775776510.301038:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776510.301041:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.301044:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.301046:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776510.301050:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776510.301051:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1775776510.301054:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776510.301055:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776510.301056:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776510.301061:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776510.301069:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776510.301070:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.301073:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776510.301075:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776510.301077:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776510.301080:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760256:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776510.301084:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776510.301085:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776510.301088:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.301089:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776510.301093:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776510.301095:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.301099:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776510.301102:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775776510.301108:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776510.301112:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099760256, portal 4 00000100:00000001:3.0:1775776510.301114:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776510.301117:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:3.0:1775776510.301123:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776510.301130:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776510.301132:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776510.301135:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1908. 00000100:00000200:3.0:1775776510.301137:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099760256, offset 0 00000400:00000200:3.0:1775776510.301142:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776510.301147:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776510.301151:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776510.301154:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776510.301157:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776510.301160:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776510.301162:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776510.301166:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4170:4170:253:4294967295] 192.168.201.139@tcp LPNI seq info [4170:4170:5:4294967295] 00000400:00000200:3.0:1775776510.301171:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776510.301178:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776510.301183:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776510.301186:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511a00. 00000800:00000200:3.0:1775776510.301190:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776510.301194:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776510.301197:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776510.301200:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776510.301203:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776510.301205:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312898063 00000100:00000001:3.0:1775776510.301207:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312899317 00000100:00000001:3.0:1775776510.301209:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312898063 00000100:00000001:3.0:1775776510.301211:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312899317 00000100:00000001:3.0:1775776510.301213:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776510.301214:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.301216:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776510.301242:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776510.301248:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776510.301249:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.301251:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776510.301253:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776510.301254:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776510.301259:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760384:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776510.301262:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776510.301265:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776510.301266:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776510.301269:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776510.301272:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776510.301274:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776510.301278:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776510.301280:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776510.301284:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776510.301288:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099760384, portal 4 00000100:00000001:3.0:1775776510.301290:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776510.301291:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:3.0:1775776510.301297:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776510.301303:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776510.301305:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776510.301308:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1f68. 00000100:00000200:3.0:1775776510.301310:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099760384, offset 0 00000400:00000200:3.0:1775776510.301315:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776510.301319:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776510.301323:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776510.301325:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776510.301328:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776510.301330:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776510.301332:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776510.301335:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4171:4171:252:4294967295] 192.168.201.139@tcp LPNI seq info [4171:4171:4:4294967295] 00000400:00000200:3.0:1775776510.301339:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776510.301344:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776510.301354:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776510.301357:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511100. 00000800:00000200:3.0:1775776510.301361:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776510.301365:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776510.301368:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776510.301371:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776510.301374:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776510.301376:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312898054 00000100:00000001:3.0:1775776510.301379:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312899317 00000100:00000001:3.0:1775776510.301381:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312898054 00000100:00000001:3.0:1775776510.301383:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312899317 00000100:00000001:3.0:1775776510.301385:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776510.301387:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.301390:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.301393:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776510.301397:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776510.301399:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1775776510.301400:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776510.301402:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776510.301404:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776510.301406:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776511.003872:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776511.003877:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776511.003879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776511.003883:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776511.003886:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776511.003889:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776511.003891:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776511.003894:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776511.003896:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776511.067848:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776511.067855:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776511.067858:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776511.067866:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776511.067873:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776511.067879:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776511.067885:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776511.067889:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776511.067892:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776511.067929:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776511.067933:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776511.067936:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776511.579900:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312899317, at 4312899328 00000100:00000040:0.0:1775776511.579907:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312899317, at 4312899328 00000100:00000010:1.0:1775776511.579916:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000100:00000010:0.0:1775776511.579919:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503449c0. 00000100:00000001:3.0:1775776512.027853:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776512.027859:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776512.027860:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776512.027865:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776512.027867:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776512.027871:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776512.027873:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776512.027875:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776512.027877:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776512.091890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776512.091897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776512.091899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776512.091907:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776512.091910:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776512.091914:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776512.091917:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776512.091919:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776512.091922:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776512.092004:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776512.092008:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776512.092012:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776513.051896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776513.051902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776513.051904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776513.051908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776513.051911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776513.051915:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.051917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776513.051919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.051922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.115848:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776513.115854:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776513.115856:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776513.115860:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776513.115865:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776513.115869:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.115873:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776513.115875:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.115878:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776513.115928:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776513.115930:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776513.115933:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775776513.372060:0:301878:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775776513.372104:0:301878:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775776513.372113:0:301878:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776513.372117:0:301878:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775776513.372122:0:301878:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776513.372125:0:301878:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775776513.372131:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776513.372134:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511e00. 00000400:00000200:1.0:1775776513.372138:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776513.372143:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776513.372148:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1ee0 00000400:00000010:1.0:1775776513.372151:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1ee0. 00000100:00000001:1.0:1775776513.372155:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776513.372164:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776513.372186:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776513.372191:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776513.372197:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776513.372199:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776513.372207:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776513.372209:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511d00. 00000400:00000200:1.0:1775776513.372211:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776513.372214:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776513.372217:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1e58 00000400:00000010:1.0:1775776513.372219:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1e58. 00000100:00000001:1.0:1775776513.372222:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776513.372226:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776513.372234:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775776513.372236:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1775776513.372237:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.372242:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776513.372246:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1775776513.372247:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776513.372249:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776513.372254:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776513.372257:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511200. 00000400:00000200:1.0:1775776513.372260:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:3.0:1775776513.372264:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776510/real 1775776513] req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776513.372267:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776513.372273:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1330 00000400:00000010:1.0:1775776513.372276:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1330. 00000100:00000001:1.0:1775776513.372280:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776513.372286:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776513.372286:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776513.372289:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776513.372294:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d580. 00000400:00000200:3.0:1775776513.372299:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1110 00000400:00000010:3.0:1775776513.372302:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1110. 00000100:00000001:3.0:1775776513.372305:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776513.372311:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776513.372318:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000200:3.0:1775776513.372322:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776513.372326:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776513.372334:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776513.372336:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776513.372344:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000100:00000001:3.0:1775776513.372347:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000010:1.0:1775776513.372348:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511a00. 00000100:00000001:3.0:1775776513.372350:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775776513.372352:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:3.0:1775776513.372355:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:1.0:1775776513.372358:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000100:00000001:3.0:1775776513.372359:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1775776513.372363:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1908 00000400:00000010:1.0:1775776513.372365:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1908. 00000100:00100000:3.0:1775776513.372366:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776513.372368:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776513.372373:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.372385:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776513.372389:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776513.372393:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776513.372395:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.372412:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776513.372415:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.372417:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776513.372418:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776513.372419:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.372421:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.372423:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000800:00000100:1.0:1775776513.372427:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000100:00000001:3.0:1775776513.372427:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776513.372431:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000800:00000010:1.0:1775776513.372432:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511100. 00000400:00000200:1.0:1775776513.372438:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00080000:3.0:1775776513.372438:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776513.372442:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:1.0:1775776513.372447:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000100:00000040:3.0:1775776513.372449:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776513.372451:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1f68 00000400:00000010:1.0:1775776513.372455:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1f68. 00000100:00000001:1.0:1775776513.372460:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1775776513.372464:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760000:192.168.201.139@tcp:38:kworker.0 00000100:00000200:1.0:1775776513.372468:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.372471:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1775776513.372477:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1775776513.372479:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776510/real 1775776513] req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776513.372483:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776535 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776513.372499:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1775776513.372500:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1775776513.372501:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775776513.372503:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:3.0:1775776513.372505:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063dec8. 00000400:00000200:3.0:1775776513.372509:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1990 00000400:00000010:3.0:1775776513.372511:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1990. 00000100:00000001:3.0:1775776513.372514:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776513.372518:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776513.372526:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.372534:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776513.372535:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.372536:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.372538:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776513.372541:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.372547:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776513.372550:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.372556:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776513.372557:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.372559:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.372560:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.372561:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776513.372562:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776513.372564:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776513.372567:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776513.372570:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776513.372573:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776513.374060:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760128:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776513.374067:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776513.374073:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776510/real 1775776513] req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776513.374085:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776513.374086:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776513.374088:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d6e0. 00000400:00000200:3.0:1775776513.374092:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776513.374094:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776513.374098:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776513.374101:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776513.374110:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374116:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776513.374118:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374121:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.374122:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776513.374126:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374133:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776513.374137:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374143:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776513.374146:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374148:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.374149:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374152:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776513.374154:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776513.374157:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776513.374161:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776513.374163:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776513.374166:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776513.374173:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099759872:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776513.374177:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.374179:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776513.374183:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dfb80 x1862033099760000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374189:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776513.374192:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776513.374195:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776513.374199:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776513.374201:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776513.374203:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776513.374205:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776513.374207:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f800. 02000000:00000001:3.0:1775776513.374211:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776513.374213:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776513.374214:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dfb80. 00000100:00000001:3.0:1775776513.374218:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776513.374219:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374221:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776513.374223:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de680 x1862033099760128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374229:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776513.374231:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776513.374232:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776513.374234:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776513.374235:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776513.374236:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776513.374237:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776513.374238:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:3.0:1775776513.374240:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776513.374241:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776513.374242:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de680. 00000100:00000001:3.0:1775776513.374244:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776513.374245:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374247:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776513.374250:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099759872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374255:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776513.374256:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776513.374258:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776513.374259:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776513.374260:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776513.374261:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776513.374263:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776513.374264:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f400. 02000000:00000001:3.0:1775776513.374266:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776513.374267:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776513.374268:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:3.0:1775776513.374270:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776513.374271:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374274:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.374276:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776513.374278:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776513.374279:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776513.374283:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776510/real 1775776513] req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776513.374291:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776513.374293:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776513.374295:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775776513.374298:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776513.374299:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776513.374301:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776513.374304:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776513.374311:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374316:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776513.374317:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374318:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.374319:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776513.374322:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374327:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776513.374338:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374344:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776513.374346:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374347:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.374348:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374350:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776513.374351:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776513.374353:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776513.374356:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776513.374358:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776513.374361:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776513.374368:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760256:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776513.374371:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776513.374375:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776510/real 1775776513] req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776513.374381:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776513.374382:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776513.374383:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776513.374385:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776513.374387:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776513.374389:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776513.374392:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776513.374399:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374405:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776513.374407:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374408:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.374410:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776513.374413:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374419:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776513.374422:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374428:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776513.374430:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374431:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776513.374432:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374433:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776513.374435:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776513.374437:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776513.374440:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776513.374442:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776513.374445:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776513.374452:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760384:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776513.374456:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374457:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776513.374459:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099760256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374464:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776513.374465:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776513.374467:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776513.374469:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776513.374470:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776513.374471:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776513.374473:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776513.374474:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e400. 02000000:00000001:3.0:1775776513.374477:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776513.374478:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776513.374479:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:3.0:1775776513.374482:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776513.374483:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374484:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776513.374487:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099760384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776535 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.374492:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776513.374493:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776513.374495:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776513.374496:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776513.374497:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776513.374498:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776513.374500:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776513.374501:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759cc00. 02000000:00000001:3.0:1775776513.374503:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776513.374504:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776513.374505:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:3.0:1775776513.374507:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776513.374508:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374510:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.374517:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776513.374519:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.374521:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776513.374522:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776514.075870:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776514.075876:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776514.075877:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776514.075882:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776514.075884:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776514.075887:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776514.075889:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776514.075891:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776514.075893:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776514.139884:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776514.139889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776514.139891:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776514.139895:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776514.139899:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776514.139903:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776514.139909:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776514.139913:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776514.139915:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776514.139984:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776514.139988:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776514.139992:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776515.099938:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776515.099945:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776515.099947:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776515.099952:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776515.099955:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776515.099958:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.099961:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776515.099963:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.099966:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.163843:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776515.163848:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776515.163850:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776515.163854:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776515.163857:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776515.163860:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.163862:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776515.163865:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.163870:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776515.163945:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776515.163947:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776515.163949:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776515.419889:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776515.419899:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776515.419902:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776515.419906:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776515.419912:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776515.419918:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18230 00000400:00000200:0.0:1775776515.419928:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776515.419933:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776515.419936:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776515.419940:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.419943:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.419945:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.419947:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776515.419951:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776515.419953:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776515.419956:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776515.419958:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776515.419960:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776515.419963:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776515.419965:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.419967:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.419969:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.419971:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776515.419975:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776515.419978:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776515.419980:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776515.419983:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776515.419986:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776515.419988:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.419991:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776515.419993:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776515.420000:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776515.420004:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776515.420008:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776515.420010:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420013:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776515.420015:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776515.420019:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776515.420026:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776515.420033:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776515.420041:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776515.420052:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776515.420057:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776515.420061:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776515.420092:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776515.420096:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776515.420099:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776515.420100:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776515.420102:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776515.420104:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776515.420107:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18230 00000400:00000200:0.0:1775776515.420110:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776515.420114:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776515.420116:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776515.420119:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420121:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.420123:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420125:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776515.420127:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776515.420128:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776515.420129:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776515.420131:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776515.420133:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776515.420135:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776515.420138:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420140:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.420141:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420143:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776515.420145:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776515.420147:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420149:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776515.420151:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776515.420156:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776515.420159:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420160:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776515.420164:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776515.420167:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420169:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776515.420172:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776515.420174:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776515.420176:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776515.420177:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420179:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776515.420181:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776515.420182:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776515.420185:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776515.420189:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776515.420205:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776515.420212:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776515.420214:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776515.420218:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776515.420225:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776515.420227:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776515.420230:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776515.420231:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776515.420232:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776515.420235:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776515.420237:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18230 00000400:00000200:0.0:1775776515.420240:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776515.420243:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776515.420245:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776515.420247:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420249:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.420251:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420253:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776515.420255:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776515.420256:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776515.420258:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776515.420259:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776515.420261:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776515.420263:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776515.420266:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420267:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.420269:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420271:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776515.420273:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776515.420275:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420278:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776515.420279:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776515.420282:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776515.420284:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420286:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776515.420288:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776515.420290:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420291:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776515.420295:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776515.420297:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776515.420299:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776515.420300:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420302:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776515.420303:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776515.420305:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776515.420308:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776515.420311:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776515.420315:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776515.420322:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776515.420324:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776515.420327:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776515.420334:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776515.420337:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776515.420340:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776515.420341:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776515.420343:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776515.420345:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776515.420347:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18230 00000400:00000200:0.0:1775776515.420351:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776515.420353:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776515.420355:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776515.420357:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420359:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.420361:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420363:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776515.420365:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776515.420366:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776515.420367:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776515.420369:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776515.420371:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776515.420373:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776515.420375:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420377:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.420378:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420380:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776515.420382:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776515.420385:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420387:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776515.420388:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776515.420391:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776515.420393:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420395:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776515.420397:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776515.420399:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420400:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776515.420404:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776515.420406:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776515.420408:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776515.420409:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420411:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776515.420412:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776515.420414:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776515.420417:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776515.420420:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776515.420424:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776515.420432:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776515.420434:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776515.420437:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776515.420444:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776515.420447:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776515.420449:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776515.420451:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776515.420452:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776515.420455:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776515.420457:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18230 00000400:00000200:0.0:1775776515.420461:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776515.420464:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776515.420466:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776515.420469:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420470:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.420473:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420475:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776515.420477:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776515.420478:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776515.420480:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776515.420481:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776515.420483:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776515.420485:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776515.420487:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420489:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776515.420491:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420492:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776515.420494:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776515.420497:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776515.420499:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776515.420500:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776515.420503:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776515.420505:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420506:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776515.420508:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776515.420510:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420511:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776515.420515:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776515.420517:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776515.420520:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776515.420521:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776515.420522:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776515.420523:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776515.420525:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776515.420528:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776515.420531:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776515.420535:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776515.420566:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776515.420568:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776515.420572:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776515.420579:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776515.420584:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18240) 00000100:00000001:3.0:1775776515.420767:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776515.420772:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776515.420774:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776515.420802:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.420810:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776515.420812:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.420815:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776515.420818:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776515.420819:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776515.420823:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099760512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776515.420827:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776515.420831:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776515.420833:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.420836:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776515.420840:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776515.420842:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776515.420846:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776515.420849:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776515.420856:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776515.420864:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099760512, portal 25 00000100:00000001:3.0:1775776515.420866:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776515.420867:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:3.0:1775776515.420872:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.420883:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776515.420886:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776515.420888:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776515.420891:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099760512, offset 0 00000400:00000200:3.0:1775776515.420898:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776515.420906:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.420911:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776515.420916:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776515.420920:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776515.420923:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776515.420926:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776515.420930:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4172:4172:256:4294967295] 192.168.201.139@tcp LPNI seq info [4172:4172:8:4294967295] 00000400:00000200:3.0:1775776515.420933:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.420943:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776515.420948:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.420953:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776515.420961:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776515.420963:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776515.420968:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776515.420972:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775776515.420991:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.420993:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776515.420997:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776515.421001:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776515.421006:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776515.421008:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776515.421011:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776515.421016:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421024:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.421027:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421038:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.421040:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776515.421045:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776515.421048:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776515.421051:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312904438 00000100:00000010:3.0:1775776515.421056:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503449c0. 00000100:00000001:3.0:1775776515.421063:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776515.421073:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312904438 00000100:00000010:3.0:1775776515.421076:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503446c0. 00000100:00000001:3.0:1775776515.421079:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776515.421081:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776515.421084:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.421086:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776515.421090:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776515/real 1775776515] req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776515.421098:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776515.421100:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776515.421102:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776515.421105:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776515.421107:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776515.421109:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776515.421112:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776515.421119:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421125:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776515.421126:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421128:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.421129:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776515.421133:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421138:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776515.421142:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421148:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776515.421150:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421152:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.421153:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421154:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776515.421156:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776515.421159:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776515.421163:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776515.421165:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776515.421168:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776515.421175:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099760512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776515.421179:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776515.421182:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421188:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776515.421189:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.421191:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776515.421193:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776515.421195:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776515.421197:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776515.421200:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776515.421202:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776515.421203:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.421204:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776515.421208:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776515.421209:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776515.421212:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776515.421214:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776515.421217:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776515.421220:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099760640, portal 10 00000100:00000001:3.0:1775776515.421221:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776515.421222:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000100:00000040:3.0:1775776515.421228:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421233:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776515.421235:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776515.421237:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776515.421239:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099760640, offset 0 00000400:00000200:3.0:1775776515.421242:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776515.421246:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.421249:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776515.421252:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776515.421255:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776515.421257:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776515.421259:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776515.421261:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4173:4173:256:4294967295] 192.168.201.139@tcp LPNI seq info [4173:4173:8:4294967295] 00000400:00000200:3.0:1775776515.421265:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.421270:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776515.421274:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.421276:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776515.421280:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776515.421282:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776515.421285:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776515.421288:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775776515.421290:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.421292:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776515.421295:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776515.421298:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776515.421300:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776515.421301:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776515.421303:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776515.421306:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421313:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.421315:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421320:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.421322:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776515.421324:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776515.421329:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776515.421332:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776515.421333:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312903173 00000100:00000001:3.0:1775776515.421336:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312904438 00000100:00000001:3.0:1775776515.421338:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312903173 00000100:00000001:3.0:1775776515.421340:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312904438 00000100:00000001:3.0:1775776515.421342:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776515.421343:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.421345:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776515.421348:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776515/real 1775776515] req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776515.421354:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776515.421355:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776515.421357:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776515.421359:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776515.421360:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776515.421362:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776515.421365:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776515.421372:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421377:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776515.421378:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421379:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.421381:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776515.421384:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421390:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776515.421393:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421399:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776515.421401:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421402:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.421404:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421405:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776515.421407:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776515.421410:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776515.421413:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776515.421415:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776515.421418:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776515.421426:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776515.421429:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776515.421432:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421443:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776515.421444:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.421446:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776515.421448:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776515.421449:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776515.421452:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776515.421454:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776515.421456:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776515.421457:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.421459:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776515.421463:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776515.421465:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776515.421468:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776515.421470:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776515.421473:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776515.421475:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099760768, portal 10 00000100:00000001:3.0:1775776515.421477:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776515.421478:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748096 : -114841967803520 : ffff978d454dc380) 00000100:00000040:3.0:1775776515.421482:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421489:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776515.421491:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776515.421493:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776515.421495:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099760768, offset 0 00000400:00000200:3.0:1775776515.421498:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776515.421502:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.421505:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776515.421508:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776515.421510:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776515.421512:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776515.421514:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776515.421517:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4174:4174:256:4294967295] 192.168.201.139@tcp LPNI seq info [4174:4174:8:4294967295] 00000400:00000200:3.0:1775776515.421520:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.421526:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776515.421530:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.421533:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776515.421537:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776515.421539:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776515.421556:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776515.421559:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775776515.421562:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.421564:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776515.421567:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776515.421570:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776515.421572:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776515.421574:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776515.421575:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776515.421578:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421583:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.421585:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421591:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.421593:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776515.421595:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776515.421598:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776515.421601:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776515.421602:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312903174 00000100:00000001:3.0:1775776515.421605:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312904438 00000100:00000001:3.0:1775776515.421607:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312903174 00000100:00000001:3.0:1775776515.421608:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312904438 00000100:00000001:3.0:1775776515.421610:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776515.421612:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.421614:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776515.421617:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776515/real 1775776515] req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776515.421625:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776515.421626:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776515.421628:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776515.421630:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776515.421632:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776515.421634:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776515.421636:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776515.421644:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421649:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776515.421650:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421651:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.421652:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776515.421654:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421659:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776515.421663:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421669:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776515.421671:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421672:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.421674:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421675:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776515.421676:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776515.421678:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776515.421683:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776515.421684:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776515.421688:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776515.421694:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776515.421698:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776515.421701:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421705:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776515.421707:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.421709:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776515.421710:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776515.421712:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776515.421715:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776515.421718:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776515.421720:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776515.421721:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.421723:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776515.421725:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776515.421727:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776515.421731:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776515.421733:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776515.421736:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776515.421738:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099760896, portal 4 00000100:00000001:3.0:1775776515.421740:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776515.421741:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741750784 : -114841967800832 : ffff978d454dce00) 00000100:00000040:3.0:1775776515.421746:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421751:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776515.421753:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776515.421755:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776515.421758:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099760896, offset 0 00000400:00000200:3.0:1775776515.421761:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776515.421765:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.421768:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776515.421771:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776515.421773:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776515.421775:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776515.421777:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776515.421809:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4175:4175:256:4294967295] 192.168.201.139@tcp LPNI seq info [4175:4175:8:4294967295] 00000400:00000200:3.0:1775776515.421812:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.421818:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776515.421822:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.421826:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776515.421829:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776515.421832:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776515.421836:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776515.421838:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775776515.421841:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.421843:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776515.421845:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776515.421849:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776515.421851:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776515.421853:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776515.421855:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776515.421858:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421864:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.421867:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421872:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.421873:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776515.421877:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776515.421880:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776515.421883:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776515.421885:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312903184 00000100:00000001:3.0:1775776515.421887:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312904438 00000100:00000001:3.0:1775776515.421889:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312903184 00000100:00000001:3.0:1775776515.421891:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312904438 00000100:00000001:3.0:1775776515.421893:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776515.421895:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.421896:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776515.421900:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776515/real 1775776515] req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776515.421908:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776515.421909:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776515.421910:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776515.421913:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776515.421914:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776515.421917:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776515.421919:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776515.421926:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421932:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776515.421933:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421935:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.421937:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776515.421940:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421945:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776515.421948:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421954:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776515.421957:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421958:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.421959:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.421961:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776515.421962:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776515.421965:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776515.421968:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776515.421970:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776515.421974:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776515.421979:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099760896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776515.421982:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776515.421984:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.421989:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776515.421990:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.421991:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776515.421993:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776515.421994:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776515.421995:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776515.421997:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776515.421999:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776515.422000:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776515.422001:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776515.422004:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776515.422005:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776515.422007:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776515.422008:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776515.422010:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776515.422012:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099761024, portal 4 00000100:00000001:3.0:1775776515.422014:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776515.422014:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748992 : -114841967802624 : ffff978d454dc700) 00000100:00000040:3.0:1775776515.422019:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422023:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776515.422025:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776515.422026:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776515.422028:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099761024, offset 0 00000400:00000200:3.0:1775776515.422030:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776515.422032:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.422034:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776515.422036:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776515.422037:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776515.422039:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776515.422040:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776515.422042:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4176:4176:256:4294967295] 192.168.201.139@tcp LPNI seq info [4176:4176:8:4294967295] 00000400:00000200:3.0:1775776515.422044:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776515.422049:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776515.422053:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.422056:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776515.422059:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776515.422061:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775776515.422065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776515.422067:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1775776515.422070:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776515.422071:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776515.422074:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776515.422076:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776515.422079:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776515.422080:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776515.422082:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776515.422086:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422093:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.422096:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776540 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422101:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.422103:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776515.422106:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776515.422109:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776515.422112:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776515.422114:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312903175 00000100:00000001:3.0:1775776515.422117:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312904438 00000100:00000001:3.0:1775776515.422118:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312903175 00000100:00000001:3.0:1775776515.422120:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312904438 00000100:00000001:3.0:1775776515.422122:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776515.422124:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.422126:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776515.422129:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776515/real 1775776515] req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776515.422136:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776515.422137:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776515.422139:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776515.422141:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776515.422142:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776515.422144:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776515.422147:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776515.422156:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422167:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776515.422169:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422170:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.422172:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776515.422174:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422178:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776515.422182:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422194:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776515.422196:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422198:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776515.422199:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422201:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776515.422202:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776515.422204:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776515.422207:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776515.422210:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776515.422213:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776515.422220:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776515.422223:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422225:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.422228:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099760512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422233:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776515.422235:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776515.422237:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776515.422240:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776515.422242:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776515.422243:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776515.422245:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776515.422247:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c000. 02000000:00000001:3.0:1775776515.422250:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776515.422252:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776515.422253:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000100:00000001:3.0:1775776515.422256:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776515.422257:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422258:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.422260:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099760640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422265:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776515.422266:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776515.422267:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776515.422269:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776515.422270:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776515.422271:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776515.422273:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776515.422274:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:3.0:1775776515.422275:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776515.422276:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776515.422277:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:3.0:1775776515.422279:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776515.422280:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422281:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.422284:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc380 x1862033099760768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422288:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776515.422289:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776515.422290:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776515.422292:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776515.422293:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776515.422294:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776515.422295:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776515.422296:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:3.0:1775776515.422299:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776515.422300:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776515.422300:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc380. 00000100:00000001:3.0:1775776515.422302:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776515.422303:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422304:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.422307:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dce00 x1862033099760896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422312:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776515.422314:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776515.422315:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776515.422316:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776515.422317:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776515.422318:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776515.422320:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776515.422321:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:3.0:1775776515.422323:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776515.422324:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776515.422325:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dce00. 00000100:00000001:3.0:1775776515.422327:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776515.422328:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422329:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776515.422331:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc700 x1862033099761024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776540 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776515.422337:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776515.422338:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776515.422339:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776515.422341:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776515.422342:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776515.422343:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776515.422345:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776515.422346:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:3.0:1775776515.422348:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776515.422349:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776515.422351:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc700. 00000100:00000001:3.0:1775776515.422352:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776515.422353:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422355:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776515.422363:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776515.422365:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776515.422368:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776515.422369:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776516.123873:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776516.123878:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776516.123880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776516.123884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776516.123886:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776516.123889:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776516.123891:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776516.123893:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776516.123896:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776516.187892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776516.187897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776516.187899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776516.187904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776516.187909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776516.187915:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776516.187919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776516.187922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776516.187924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776516.187962:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776516.187965:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776516.187967:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776516.699875:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312904438, at 4312904448 00000100:00000010:1.0:1775776516.699896:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000100:00000040:0.0:1775776516.699972:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312904438, at 4312904448 00000100:00000010:0.0:1775776516.699980:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503449c0. 00000100:00000001:3.0:1775776517.147899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776517.147905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776517.147906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776517.147910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776517.147913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776517.147917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776517.147919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776517.147920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776517.147922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776517.211847:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776517.211855:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776517.211857:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776517.211865:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776517.211870:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776517.211874:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776517.211877:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776517.211879:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776517.211884:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776517.211943:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776517.211947:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776517.211950:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776518.171880:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776518.171885:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776518.171886:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776518.171890:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776518.171893:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776518.171896:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776518.171897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776518.171899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776518.171901:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776518.235914:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776518.235920:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776518.235923:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776518.235931:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776518.235935:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776518.235939:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776518.235941:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776518.235943:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776518.235946:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776518.236029:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776518.236032:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776518.236035:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776519.196046:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776519.196051:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776519.196053:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776519.196056:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776519.196059:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776519.196062:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776519.196064:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776519.196066:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776519.196067:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776519.259896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776519.259904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776519.259906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776519.259911:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776519.259914:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776519.259917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776519.259920:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776519.259922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776519.259925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776519.260117:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776519.260120:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776519.260123:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776520.219905:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776520.219913:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776520.219914:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776520.219920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776520.219924:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776520.219928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.219930:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776520.219933:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.219935:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.283887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776520.283894:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776520.283896:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776520.283901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776520.283906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776520.283911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.283915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776520.283918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.283922:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776520.284069:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776520.284072:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776520.284075:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776520.540837:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776520.540841:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776520.540843:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776520.540849:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.540856:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776520.540858:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.540861:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776520.540863:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776520.540865:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776520.540868:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099761152:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776520.540872:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776520.540875:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776520.540878:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.540880:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776520.540885:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776520.540887:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776520.540892:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776520.540896:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776520.540902:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776520.540909:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099761152, portal 25 00000100:00000001:3.0:1775776520.540911:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776520.540912:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288714240 : -114841420837376 : ffff978d65e7ce00) 00000100:00000040:3.0:1775776520.540917:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.540933:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776520.540936:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776520.540939:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776520.540943:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099761152, offset 0 00000400:00000200:3.0:1775776520.540949:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776520.540956:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.540961:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776520.540966:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776520.540970:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776520.540973:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776520.540975:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776520.540979:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4177:4177:256:4294967295] 192.168.201.139@tcp LPNI seq info [4177:4177:8:4294967295] 00000400:00000200:3.0:1775776520.540983:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.540992:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776520.540997:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.541002:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776520.541008:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776520.541011:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775776520.541016:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776520.541020:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775776520.541024:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.541026:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776520.541032:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776520.541036:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776520.541041:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776520.541043:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776520.541046:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776520.541050:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541243:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.541247:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541253:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.541255:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776520.541261:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776520.541264:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776520.541267:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312909558 00000100:00000010:3.0:1775776520.541271:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344d20. 00000100:00000001:3.0:1775776520.541277:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776520.541283:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312909558 00000100:00000010:3.0:1775776520.541285:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344ba0. 00000100:00000001:3.0:1775776520.541289:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776520.541292:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776520.541294:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.541296:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776520.541300:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776520/real 1775776520] req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776520.541309:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776520.541310:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776520.541312:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776520.541314:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776520.541316:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776520.541318:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776520.541321:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776520.541328:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541334:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776520.541335:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541337:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.541339:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776520.541342:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541349:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776520.541353:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541360:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776520.541362:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541364:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.541365:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541367:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776520.541369:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776520.541371:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776520.541375:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776520.541377:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776520.541381:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776520.541389:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099761152:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776520.541394:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541395:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.541398:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ce00 x1862033099761152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541410:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776520.541412:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776520.541414:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776520.541417:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776520.541418:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776520.541419:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776520.541421:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776520.541424:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:3.0:1775776520.541427:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776520.541429:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776520.541431:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ce00. 00000100:00000001:3.0:1775776520.541433:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776520.541434:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541436:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541443:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776520.541445:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.541447:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776520.541448:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776520.541449:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776520.541465:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541472:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776520.541473:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.541475:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776520.541477:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776520.541479:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776520.541481:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761280:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776520.541484:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776520.541485:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776520.541487:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.541488:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776520.541491:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776520.541493:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776520.541496:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776520.541499:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776520.541502:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776520.541505:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099761280, portal 10 00000100:00000001:3.0:1775776520.541507:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776520.541508:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288715136 : -114841420836480 : ffff978d65e7d180) 00000100:00000040:3.0:1775776520.541514:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541520:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776520.541522:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776520.541524:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776520.541527:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099761280, offset 0 00000400:00000200:3.0:1775776520.541530:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776520.541534:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.541538:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776520.541540:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776520.541541:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776520.541543:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776520.541544:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776520.541546:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4178:4178:256:4294967295] 192.168.201.139@tcp LPNI seq info [4178:4178:8:4294967295] 00000400:00000200:3.0:1775776520.541548:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.541552:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776520.541555:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.541557:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776520.541560:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776520.541561:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775776520.541564:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776520.541565:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775776520.541568:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.541569:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776520.541571:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776520.541573:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776520.541575:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776520.541576:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776520.541577:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776520.541580:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541584:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.541586:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541590:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.541591:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776520.541593:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776520.541595:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776520.541597:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776520.541598:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312908293 00000100:00000001:3.0:1775776520.541600:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312909558 00000100:00000001:3.0:1775776520.541601:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312908293 00000100:00000001:3.0:1775776520.541602:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312909558 00000100:00000001:3.0:1775776520.541603:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776520.541605:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.541606:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776520.541607:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776520/real 1775776520] req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776520.541612:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776520.541613:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776520.541614:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776520.541615:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776520.541616:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776520.541617:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776520.541619:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776520.541624:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541630:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776520.541631:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541632:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.541633:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776520.541635:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541638:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776520.541640:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541643:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776520.541645:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541646:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.541646:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541647:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776520.541648:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776520.541650:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776520.541651:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776520.541652:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776520.541654:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776520.541658:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761280:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776520.541661:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541662:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.541665:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d180 x1862033099761280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541671:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776520.541672:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776520.541674:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776520.541676:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776520.541677:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776520.541678:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776520.541680:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776520.541681:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505400. 02000000:00000001:3.0:1775776520.541683:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776520.541684:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776520.541686:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d180. 00000100:00000001:3.0:1775776520.541688:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776520.541689:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541691:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.541694:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776520.541695:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.541697:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776520.541698:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.541763:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776520.541766:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.541891:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776520.541895:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776520.541899:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776520.541907:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.541924:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776520.541927:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.541931:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776520.541935:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776520.541939:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776520.541944:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761408:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776520.541951:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776520.541954:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776520.541958:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.541962:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776520.541967:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776520.542061:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776520.542069:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776520.542073:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776520.542079:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776520.542084:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099761408, portal 10 00000100:00000001:3.0:1775776520.542087:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776520.542090:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288713344 : -114841420838272 : ffff978d65e7ca80) 00000100:00000040:3.0:1775776520.542099:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542111:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776520.542116:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776520.542121:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776520.542125:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099761408, offset 0 00000400:00000200:3.0:1775776520.542134:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776520.542142:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.542150:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776520.542155:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776520.542159:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776520.542163:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776520.542166:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776520.542169:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4179:4179:256:4294967295] 192.168.201.139@tcp LPNI seq info [4179:4179:8:4294967295] 00000400:00000200:3.0:1775776520.542174:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.542181:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776520.542185:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.542189:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776520.542193:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776520.542196:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775776520.542200:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776520.542202:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775776520.542206:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.542208:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776520.542211:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776520.542214:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776520.542217:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776520.542220:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776520.542222:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776520.542226:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542234:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.542237:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542244:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.542245:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776520.542248:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776520.542251:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776520.542255:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776520.542257:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312908295 00000100:00000001:3.0:1775776520.542259:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312909558 00000100:00000001:3.0:1775776520.542261:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312908295 00000100:00000001:3.0:1775776520.542263:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312909558 00000100:00000001:3.0:1775776520.542265:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776520.542267:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.542269:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776520.542273:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776520/real 1775776520] req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776520.542281:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776520.542283:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776520.542284:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776520.542287:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776520.542288:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776520.542290:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776520.542293:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776520.542301:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542318:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776520.542319:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542321:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.542323:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776520.542326:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542332:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776520.542336:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542342:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776520.542344:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542346:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.542347:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542348:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776520.542350:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776520.542352:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776520.542356:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776520.542358:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776520.542361:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776520.542369:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761408:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776520.542373:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542375:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.542378:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ca80 x1862033099761408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542384:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776520.542386:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776520.542388:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776520.542391:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776520.542393:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776520.542394:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776520.542396:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776520.542397:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506800. 02000000:00000001:3.0:1775776520.542400:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776520.542401:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776520.542402:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ca80. 00000100:00000001:3.0:1775776520.542404:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776520.542406:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542407:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542410:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776520.542412:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.542414:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776520.542415:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776520.542416:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776520.542419:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542426:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776520.542427:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.542429:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776520.542431:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776520.542433:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776520.542435:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761536:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776520.542439:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776520.542440:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776520.542442:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.542444:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776520.542446:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776520.542448:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776520.542468:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776520.542472:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776520.542475:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776520.542478:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099761536, portal 4 00000100:00000001:3.0:1775776520.542480:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776520.542481:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288715136 : -114841420836480 : ffff978d65e7d180) 00000100:00000040:3.0:1775776520.542487:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542493:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776520.542495:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776520.542498:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776520.542500:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099761536, offset 0 00000400:00000200:3.0:1775776520.542504:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776520.542508:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.542512:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776520.542515:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776520.542517:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776520.542520:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776520.542522:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776520.542525:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4180:4180:256:4294967295] 192.168.201.139@tcp LPNI seq info [4180:4180:8:4294967295] 00000400:00000200:3.0:1775776520.542529:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.542535:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776520.542539:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.542542:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776520.542546:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776520.542549:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775776520.542553:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776520.542556:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775776520.542559:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.542561:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776520.542564:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776520.542567:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776520.542570:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776520.542572:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776520.542574:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776520.542578:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542590:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.542593:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542599:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.542600:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776520.542603:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776520.542606:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776520.542609:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776520.542611:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312908304 00000100:00000001:3.0:1775776520.542613:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312909558 00000100:00000001:3.0:1775776520.542615:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312908304 00000100:00000001:3.0:1775776520.542617:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312909558 00000100:00000001:3.0:1775776520.542619:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776520.542622:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.542623:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776520.542626:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776520/real 1775776520] req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776520.542634:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776520.542635:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776520.542637:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776520.542639:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776520.542640:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776520.542642:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776520.542645:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776520.542653:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542658:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776520.542660:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542662:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.542663:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776520.542667:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542673:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776520.542676:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542683:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776520.542685:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542687:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.542688:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542689:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776520.542691:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776520.542693:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776520.542696:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776520.542698:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776520.542702:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776520.542709:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761536:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776520.542713:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776520.542716:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542722:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776520.542723:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.542725:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776520.542727:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776520.542729:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776520.542732:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761664:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776520.542735:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776520.542737:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776520.542739:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776520.542741:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776520.542744:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776520.542745:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776520.542761:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776520.542764:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776520.542767:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776520.542770:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099761664, portal 4 00000100:00000001:3.0:1775776520.542772:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776520.542773:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288714240 : -114841420837376 : ffff978d65e7ce00) 00000100:00000040:3.0:1775776520.542803:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542810:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776520.542812:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776520.542814:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776520.542817:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099761664, offset 0 00000400:00000200:3.0:1775776520.542821:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776520.542825:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.542828:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776520.542830:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776520.542833:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776520.542835:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776520.542837:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776520.542839:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4181:4181:256:4294967295] 192.168.201.139@tcp LPNI seq info [4181:4181:8:4294967295] 00000400:00000200:3.0:1775776520.542843:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776520.542850:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776520.542853:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.542856:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776520.542861:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776520.542863:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775776520.542867:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776520.542869:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775776520.542872:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776520.542874:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776520.542877:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776520.542880:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776520.542882:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776520.542884:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776520.542885:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776520.542889:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542896:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.542898:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776545 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542903:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.542905:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776520.542907:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776520.542909:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776520.542912:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776520.542914:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312908296 00000100:00000001:3.0:1775776520.542916:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312909558 00000100:00000001:3.0:1775776520.542918:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312908296 00000100:00000001:3.0:1775776520.542920:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312909558 00000100:00000001:3.0:1775776520.542921:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776520.542923:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.542925:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776520.542928:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776520/real 1775776520] req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776520.542935:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776520.542937:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776520.542939:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776520.542941:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776520.542942:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776520.542945:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776520.542948:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776520.542956:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542962:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776520.542963:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542965:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.542966:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776520.542970:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542986:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776520.542990:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.542996:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776520.542998:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.542999:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776520.543001:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.543002:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776520.543004:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776520.543007:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776520.543010:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776520.543013:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776520.543016:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776520.543030:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761664:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776520.543033:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.543035:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.543037:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d180 x1862033099761536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.543043:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776520.543045:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776520.543046:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776520.543049:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776520.543051:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776520.543052:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776520.543054:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776520.543056:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507400. 02000000:00000001:3.0:1775776520.543059:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776520.543060:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776520.543061:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d180. 00000100:00000001:3.0:1775776520.543063:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776520.543064:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.543066:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776520.543068:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ce00 x1862033099761664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776545 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776520.543074:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776520.543075:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776520.543076:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776520.543078:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776520.543079:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776520.543080:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776520.543082:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776520.543083:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507000. 02000000:00000001:3.0:1775776520.543085:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776520.543086:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776520.543087:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ce00. 00000100:00000001:3.0:1775776520.543090:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776520.543091:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.543093:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776520.543098:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776520.543099:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776520.543100:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776520.543102:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776521.243907:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776521.243913:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776521.243915:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776521.243920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776521.243927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776521.243930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776521.243933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776521.243935:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776521.243937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776521.307999:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776521.308003:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776521.308007:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776521.308044:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776521.308048:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776521.308050:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776521.308053:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776521.308057:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776521.308060:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776521.308062:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776521.308065:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776521.308068:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776521.819880:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312909558, at 4312909568 00000100:00000010:1.0:1775776521.819893:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000100:00000040:0.0:1775776521.819980:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312909558, at 4312909568 00000100:00000010:0.0:1775776521.819987:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344d20. 00000100:00000001:3.0:1775776522.267868:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776522.267874:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776522.267876:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776522.267882:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776522.267885:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776522.267888:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776522.267891:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776522.267893:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776522.267895:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776522.331923:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776522.331933:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776522.331936:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776522.331945:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776522.331955:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776522.331961:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776522.331966:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776522.331969:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776522.331972:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776522.332020:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776522.332023:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776522.332025:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776523.291916:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776523.291924:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776523.291926:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776523.291933:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776523.291936:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776523.291940:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776523.291942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776523.291945:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776523.291947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776523.355940:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776523.355949:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776523.355953:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776523.355962:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776523.355968:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776523.355974:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776523.355979:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776523.355983:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776523.355986:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776523.356046:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776523.356051:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776523.356055:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776524.315918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776524.315925:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776524.315927:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776524.315933:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776524.315936:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776524.315940:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776524.315942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776524.315944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776524.315946:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776524.379917:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776524.379925:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776524.379929:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776524.379938:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776524.379943:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776524.379949:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776524.379953:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776524.379956:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776524.379959:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776524.380045:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776524.380049:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776524.380053:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776525.339929:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776525.339939:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776525.339941:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776525.339948:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776525.339953:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776525.339957:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.339960:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776525.339963:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.339967:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.403926:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776525.403934:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776525.403938:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776525.403946:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776525.403951:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776525.403956:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.403959:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776525.403964:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.403977:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776525.404030:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776525.404033:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776525.404036:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776525.659948:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776525.659961:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776525.659963:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776525.659968:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776525.659974:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776525.659982:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18240 00000400:00000200:1.0:1775776525.659995:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776525.660003:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776525.660007:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776525.660011:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660014:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660017:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660020:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776525.660025:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776525.660027:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776525.660030:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776525.660033:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776525.660036:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776525.660038:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776525.660041:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660043:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660045:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660046:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776525.660048:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776525.660051:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660053:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776525.660056:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776525.660059:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776525.660061:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660062:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776525.660064:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776525.660075:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776525.660078:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776525.660082:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776525.660083:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660086:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776525.660089:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776525.660094:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776525.660103:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776525.660111:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776525.660130:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776525.660141:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776525.660146:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776525.660149:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776525.660172:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776525.660175:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776525.660177:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776525.660179:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776525.660180:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776525.660182:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776525.660183:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18240 00000400:00000200:1.0:1775776525.660187:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776525.660188:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776525.660190:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776525.660192:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660193:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660195:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660196:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776525.660197:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776525.660198:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776525.660199:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776525.660200:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776525.660201:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776525.660202:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776525.660204:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660205:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660206:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660208:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776525.660210:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776525.660212:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660214:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776525.660216:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776525.660223:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776525.660225:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660227:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776525.660231:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776525.660234:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660236:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776525.660240:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776525.660242:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776525.660245:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776525.660246:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660247:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776525.660249:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776525.660251:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776525.660255:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776525.660259:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776525.660264:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776525.660272:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776525.660274:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776525.660277:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776525.660283:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776525.660285:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776525.660288:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776525.660289:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776525.660290:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776525.660292:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776525.660294:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18240 00000400:00000200:1.0:1775776525.660297:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776525.660299:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776525.660300:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776525.660302:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660304:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660305:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660306:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776525.660308:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776525.660308:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776525.660309:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776525.660310:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776525.660311:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776525.660312:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776525.660314:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660315:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660316:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660317:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776525.660318:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776525.660320:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660322:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776525.660323:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776525.660325:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776525.660326:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660327:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776525.660328:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776525.660330:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660331:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776525.660333:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776525.660335:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776525.660337:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776525.660338:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660339:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776525.660340:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776525.660341:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776525.660343:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776525.660345:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776525.660348:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776525.660353:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776525.660354:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776525.660356:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776525.660362:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776525.660374:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776525.660377:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776525.660378:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776525.660380:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776525.660382:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776525.660385:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18240 00000400:00000200:1.0:1775776525.660389:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776525.660391:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776525.660394:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776525.660396:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660398:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660400:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660402:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776525.660404:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776525.660405:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776525.660407:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776525.660410:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776525.660412:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776525.660414:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776525.660417:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660419:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660420:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660422:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776525.660424:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776525.660426:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660429:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776525.660430:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776525.660432:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776525.660434:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660435:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776525.660437:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776525.660439:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660440:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776525.660443:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776525.660444:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776525.660445:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776525.660446:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660447:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776525.660448:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776525.660450:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776525.660452:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776525.660455:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776525.660458:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776525.660462:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776525.660464:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776525.660466:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776525.660483:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776525.660486:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776525.660489:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776525.660490:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776525.660492:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776525.660494:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776525.660497:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18240 00000400:00000200:1.0:1775776525.660502:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776525.660504:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776525.660507:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776525.660509:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660511:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660513:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660515:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776525.660517:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776525.660518:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776525.660520:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776525.660521:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776525.660523:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776525.660532:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776525.660535:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660538:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776525.660539:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660541:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776525.660543:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776525.660545:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776525.660547:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776525.660550:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776525.660552:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776525.660554:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660556:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776525.660558:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776525.660561:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660562:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776525.660566:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776525.660569:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776525.660571:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776525.660572:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776525.660574:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776525.660575:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776525.660577:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776525.660581:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776525.660584:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776525.660588:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776525.660595:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776525.660597:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776525.660599:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776525.660604:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776525.660608:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18251) 00000100:00000001:3.0:1775776525.660673:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776525.660677:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776525.660679:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776525.660684:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.660690:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776525.660692:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.660695:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776525.660697:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776525.660699:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776525.660702:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099761792:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776525.660706:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776525.660709:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776525.660711:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.660713:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776525.660718:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776525.660719:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776525.660724:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776525.660727:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776525.660734:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776525.660743:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099761792, portal 25 00000100:00000001:3.0:1775776525.660745:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776525.660747:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287596544 : -114841421955072 : ffff978d65d6c000) 00000100:00000040:3.0:1775776525.660752:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.660759:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776525.660762:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776525.660764:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776525.660769:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099761792, offset 0 00000400:00000200:3.0:1775776525.660776:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776525.660806:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.660811:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776525.660815:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776525.660818:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776525.660822:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776525.660825:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776525.660830:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4182:4182:256:4294967295] 192.168.201.139@tcp LPNI seq info [4182:4182:8:4294967295] 00000400:00000200:3.0:1775776525.660835:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.660847:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776525.660863:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.660870:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776525.660881:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776525.660885:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775776525.660891:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776525.660895:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1775776525.660899:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.660902:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776525.660910:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776525.660916:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776525.660922:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776525.660924:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776525.660929:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776525.660933:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.660943:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.660946:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.660954:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.660957:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776525.660962:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776525.660967:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776525.660970:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312914678 00000100:00000010:3.0:1775776525.660976:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344c00. 00000100:00000001:3.0:1775776525.660984:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776525.660991:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312914678 00000100:00000010:3.0:1775776525.660994:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503440c0. 00000100:00000001:3.0:1775776525.660997:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776525.661001:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776525.661003:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.661006:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776525.661010:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776525/real 1775776525] req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776525.661021:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776525.661023:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776525.661025:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776525.661028:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776525.661030:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776525.661032:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776525.661035:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776525.661048:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661053:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776525.661054:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661056:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.661058:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776525.661061:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661065:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776525.661068:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661072:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776525.661074:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661075:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.661076:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661078:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776525.661080:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776525.661082:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776525.661085:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776525.661086:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776525.661088:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776525.661092:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099761792:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776525.661095:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776525.661097:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661100:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776525.661102:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.661103:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776525.661104:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776525.661106:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776525.661107:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761920:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776525.661109:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776525.661110:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776525.661111:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.661112:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776525.661114:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776525.661116:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776525.661119:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776525.661122:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776525.661125:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776525.661128:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099761920, portal 10 00000100:00000001:3.0:1775776525.661130:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776525.661131:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609984 : -114841421941632 : ffff978d65d6f480) 00000100:00000040:3.0:1775776525.661137:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661143:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776525.661145:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776525.661147:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776525.661150:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099761920, offset 0 00000400:00000200:3.0:1775776525.661154:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776525.661158:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.661162:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776525.661165:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776525.661167:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776525.661170:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776525.661172:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776525.661175:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4183:4183:256:4294967295] 192.168.201.139@tcp LPNI seq info [4183:4183:8:4294967295] 00000400:00000200:3.0:1775776525.661179:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.661185:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776525.661190:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.661193:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776525.661196:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776525.661199:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775776525.661203:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776525.661205:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1775776525.661209:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.661210:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776525.661213:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776525.661217:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776525.661220:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776525.661222:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776525.661224:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776525.661228:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661235:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.661238:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661244:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.661246:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776525.661248:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776525.661252:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776525.661255:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776525.661257:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312913413 00000100:00000001:3.0:1775776525.661260:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312914678 00000100:00000001:3.0:1775776525.661262:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312913413 00000100:00000001:3.0:1775776525.661263:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312914678 00000100:00000001:3.0:1775776525.661264:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776525.661266:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.661267:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776525.661270:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776525/real 1775776525] req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776525.661276:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776525.661277:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776525.661278:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776525.661279:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776525.661280:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776525.661282:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776525.661283:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776525.661289:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661292:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776525.661293:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661294:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.661295:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776525.661296:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661300:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776525.661302:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661305:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776525.661306:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661307:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.661308:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661309:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776525.661310:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776525.661312:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776525.661314:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776525.661315:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776525.661318:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776525.661323:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099761920:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776525.661325:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776525.661327:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661330:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776525.661331:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.661332:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776525.661333:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776525.661334:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776525.661335:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762048:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776525.661337:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776525.661338:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776525.661339:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.661340:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776525.661343:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776525.661344:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776525.661346:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776525.661347:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776525.661349:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776525.661351:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099762048, portal 10 00000100:00000001:3.0:1775776525.661352:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776525.661353:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287608192 : -114841421943424 : ffff978d65d6ed80) 00000100:00000040:3.0:1775776525.661357:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661375:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776525.661377:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776525.661380:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776525.661382:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099762048, offset 0 00000400:00000200:3.0:1775776525.661387:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776525.661391:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.661394:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776525.661396:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776525.661399:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776525.661401:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776525.661403:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776525.661405:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4184:4184:256:4294967295] 192.168.201.139@tcp LPNI seq info [4184:4184:8:4294967295] 00000400:00000200:3.0:1775776525.661409:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.661414:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776525.661418:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.661421:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776525.661424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776525.661427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775776525.661431:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776525.661433:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1775776525.661454:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.661456:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776525.661458:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776525.661460:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776525.661462:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776525.661464:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776525.661466:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776525.661469:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661475:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.661477:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661481:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.661483:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776525.661485:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776525.661487:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776525.661490:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776525.661492:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312913414 00000100:00000001:3.0:1775776525.661494:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312914678 00000100:00000001:3.0:1775776525.661496:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312913414 00000100:00000001:3.0:1775776525.661497:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312914678 00000100:00000001:3.0:1775776525.661499:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776525.661500:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.661501:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776525.661503:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776525/real 1775776525] req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776525.661508:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776525.661509:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776525.661510:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776525.661511:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776525.661512:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776525.661513:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776525.661514:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776525.661519:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661523:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776525.661524:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661525:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.661526:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776525.661527:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661533:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776525.661536:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661542:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776525.661545:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661546:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.661547:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661549:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776525.661550:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776525.661552:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776525.661556:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776525.661558:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776525.661562:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776525.661570:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762048:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776525.661574:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776525.661577:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661584:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776525.661585:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.661587:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776525.661589:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776525.661591:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776525.661594:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762176:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776525.661597:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776525.661599:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776525.661601:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.661602:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776525.661605:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776525.661607:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776525.661611:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776525.661613:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776525.661617:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776525.661619:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099762176, portal 4 00000100:00000001:3.0:1775776525.661622:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776525.661623:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287600128 : -114841421951488 : ffff978d65d6ce00) 00000100:00000040:3.0:1775776525.661629:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661634:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776525.661636:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776525.661638:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776525.661641:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099762176, offset 0 00000400:00000200:3.0:1775776525.661644:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776525.661648:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.661651:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776525.661654:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776525.661656:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776525.661659:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776525.661661:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776525.661664:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4185:4185:256:4294967295] 192.168.201.139@tcp LPNI seq info [4185:4185:8:4294967295] 00000400:00000200:3.0:1775776525.661668:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.661673:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776525.661677:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.661680:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776525.661684:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776525.661686:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775776525.661690:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776525.661693:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1775776525.661696:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.661697:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776525.661700:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776525.661703:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776525.661706:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776525.661707:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776525.661710:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776525.661713:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661725:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.661728:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661734:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.661736:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776525.661739:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776525.661741:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776525.661744:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776525.661746:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312913423 00000100:00000001:3.0:1775776525.661748:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312914678 00000100:00000001:3.0:1775776525.661750:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312913423 00000100:00000001:3.0:1775776525.661753:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312914678 00000100:00000001:3.0:1775776525.661755:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776525.661757:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.661758:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776525.661761:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776525/real 1775776525] req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776525.661769:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776525.661770:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776525.661771:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776525.661774:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776525.661775:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776525.661800:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776525.661804:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776525.661813:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661820:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776525.661822:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661824:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.661825:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776525.661828:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661835:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776525.661839:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661846:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776525.661848:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661850:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.661851:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.661853:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776525.661855:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776525.661856:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776525.661859:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776525.661861:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776525.661864:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776525.661871:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762176:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776525.661875:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776525.661878:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661884:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776525.661886:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.661888:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776525.661889:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776525.661891:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776525.661894:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762304:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776525.661897:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776525.661899:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776525.661901:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776525.661902:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776525.661906:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776525.661907:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776525.661911:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776525.661913:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776525.661916:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776525.661918:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099762304, portal 4 00000100:00000001:3.0:1775776525.661920:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776525.661921:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601024 : -114841421950592 : ffff978d65d6d180) 00000100:00000040:3.0:1775776525.661926:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.661932:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776525.661934:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776525.661936:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776525.661939:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099762304, offset 0 00000400:00000200:3.0:1775776525.661942:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776525.661946:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.661950:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776525.661952:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776525.661954:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776525.661957:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776525.661959:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776525.661962:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4186:4186:256:4294967295] 192.168.201.139@tcp LPNI seq info [4186:4186:8:4294967295] 00000400:00000200:3.0:1775776525.661966:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776525.661973:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776525.661977:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.661981:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776525.661985:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776525.661988:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775776525.661992:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776525.661994:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1775776525.661998:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776525.662000:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776525.662003:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776525.662005:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776525.662008:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776525.662009:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776525.662012:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776525.662015:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662022:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.662024:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776550 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662031:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.662032:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776525.662035:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776525.662038:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776525.662041:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776525.662043:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312913415 00000100:00000001:3.0:1775776525.662046:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312914678 00000100:00000001:3.0:1775776525.662048:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312913415 00000100:00000001:3.0:1775776525.662050:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312914678 00000100:00000001:3.0:1775776525.662052:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776525.662054:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.662056:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776525.662059:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776525/real 1775776525] req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776525.662067:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776525.662069:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776525.662071:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776525.662073:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776525.662074:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776525.662076:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776525.662079:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776525.662087:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662093:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776525.662094:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662096:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.662097:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776525.662101:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662108:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776525.662111:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662117:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776525.662120:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662122:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776525.662123:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662125:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776525.662126:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776525.662129:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776525.662132:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776525.662134:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776525.662138:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776525.662151:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762304:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776525.662155:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662157:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.662160:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c000 x1862033099761792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662167:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776525.662170:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776525.662172:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776525.662175:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776525.662177:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776525.662179:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776525.662181:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776525.662183:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39000. 02000000:00000001:3.0:1775776525.662187:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776525.662189:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776525.662191:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c000. 00000100:00000001:3.0:1775776525.662194:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776525.662195:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662196:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.662199:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f480 x1862033099761920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662203:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776525.662204:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776525.662205:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776525.662206:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776525.662207:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776525.662207:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776525.662208:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776525.662209:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3bc00. 02000000:00000001:3.0:1775776525.662210:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776525.662211:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776525.662211:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f480. 00000100:00000001:3.0:1775776525.662213:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776525.662214:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662215:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.662217:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ed80 x1862033099762048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662221:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776525.662221:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776525.662222:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776525.662223:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776525.662224:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776525.662224:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776525.662225:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776525.662226:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b800. 02000000:00000001:3.0:1775776525.662227:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776525.662227:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776525.662228:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ed80. 00000100:00000001:3.0:1775776525.662229:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776525.662230:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662231:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.662232:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ce00 x1862033099762176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662235:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776525.662236:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776525.662236:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776525.662238:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776525.662238:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776525.662239:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776525.662240:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776525.662240:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a400. 02000000:00000001:3.0:1775776525.662242:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776525.662242:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776525.662243:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ce00. 00000100:00000001:3.0:1775776525.662245:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776525.662246:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662247:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776525.662250:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d180 x1862033099762304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776550 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776525.662256:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776525.662257:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776525.662258:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776525.662260:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776525.662261:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776525.662262:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776525.662264:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776525.662265:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39400. 02000000:00000001:3.0:1775776525.662267:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776525.662268:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776525.662269:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d180. 00000100:00000001:3.0:1775776525.662271:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776525.662272:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662275:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776525.662283:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776525.662285:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776525.662287:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776525.662289:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776526.363939:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776526.363948:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776526.363950:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776526.363957:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776526.363961:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776526.363966:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776526.363969:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776526.363971:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776526.363974:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776526.427972:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776526.427980:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776526.427982:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776526.427989:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776526.427993:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776526.427997:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776526.427999:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776526.428001:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776526.428004:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776526.430539:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776526.430543:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776526.430547:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776526.939926:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312914678, at 4312914688 00000100:00000010:1.0:1775776526.939945:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503440c0. 00000100:00000040:0.0:1775776526.940051:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312914678, at 4312914688 00000100:00000010:0.0:1775776526.940059:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000100:00000001:3.0:1775776527.387937:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776527.387945:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776527.387947:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776527.387953:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776527.387957:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776527.387963:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776527.387966:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776527.387969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776527.387972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776527.451960:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776527.451971:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776527.451976:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776527.451986:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776527.451993:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776527.452001:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776527.452005:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776527.452009:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776527.452017:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776527.452093:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776527.452096:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776527.452098:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776528.411903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776528.411909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776528.411911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776528.411925:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776528.411929:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776528.411933:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776528.411936:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776528.411938:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776528.411941:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776528.475981:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776528.475984:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776528.475987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776528.476023:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776528.476027:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776528.476028:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776528.476031:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776528.476034:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776528.476036:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776528.476038:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776528.476040:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776528.476042:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776529.435934:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776529.435942:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776529.435945:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776529.435950:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776529.435953:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776529.435956:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776529.435959:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776529.435962:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776529.435965:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776529.499947:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776529.499952:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776529.499954:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776529.499958:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776529.499962:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776529.499966:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776529.499969:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776529.499971:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776529.499973:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776529.499975:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776529.499979:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776529.499983:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776530.459895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776530.459900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776530.459902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776530.459905:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776530.459908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776530.459911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.459912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776530.459914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.459916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.523884:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776530.523889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776530.523892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776530.523897:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776530.523902:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776530.523906:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.523910:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776530.523913:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.523917:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776530.524016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776530.524021:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776530.524024:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776530.780094:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776530.780099:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776530.780103:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776530.780113:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780121:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776530.780122:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.780133:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776530.780136:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776530.780139:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776530.780143:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099762432:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776530.780149:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776530.780153:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776530.780155:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.780160:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776530.780165:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776530.780167:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776530.780173:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776530.780181:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776530.780188:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776530.780195:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099762432, portal 25 00000100:00000001:3.0:1775776530.780198:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776530.780200:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288721408 : -114841420830208 : ffff978d65e7ea00) 00000100:00000040:3.0:1775776530.780206:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780223:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776530.780229:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776530.780234:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776530.780240:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099762432, offset 0 00000400:00000200:3.0:1775776530.780252:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776530.780264:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.780301:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776530.780308:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776530.780313:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776530.780316:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776530.780319:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776530.780323:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4187:4187:256:4294967295] 192.168.201.139@tcp LPNI seq info [4187:4187:8:4294967295] 00000400:00000200:3.0:1775776530.780328:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.780345:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776530.780358:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.780372:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776530.780384:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776530.780389:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776530.780400:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776530.780402:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775776530.780408:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.780411:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776530.780420:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776530.780430:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776530.780439:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776530.780444:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776530.780450:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776530.780456:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780467:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.780471:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780477:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.780479:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776530.780484:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776530.780491:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776530.780495:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312919797 00000100:00000010:3.0:1775776530.780504:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344ae0. 00000100:00000001:3.0:1775776530.780513:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776530.780530:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312919797 00000100:00000010:3.0:1775776530.780536:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344600. 00000100:00000001:3.0:1775776530.780541:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776530.780548:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776530.780553:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.780558:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776530.780567:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776530/real 1775776530] req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776530.780588:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776530.780591:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776530.780595:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776530.780599:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776530.780601:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776530.780605:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776530.780610:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776530.780639:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780653:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776530.780657:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.780663:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.780666:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776530.780673:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780686:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776530.780696:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780711:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776530.780717:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.780720:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.780722:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.780726:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776530.780728:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776530.780731:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776530.780738:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776530.780744:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776530.780751:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776530.780769:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099762432:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776530.780810:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.780815:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.780822:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ea00 x1862033099762432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780838:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776530.780842:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776530.780845:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776530.780851:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776530.780855:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776530.780859:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776530.780864:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776530.780868:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:3.0:1775776530.780880:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776530.780883:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776530.780886:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ea00. 00000100:00000001:3.0:1775776530.780891:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776530.780894:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.780897:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.780907:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776530.780910:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.780915:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776530.780917:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776530.780920:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776530.780926:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.780951:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776530.780954:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.780958:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776530.780961:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776530.780967:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776530.780974:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762560:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776530.780980:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776530.780984:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776530.780987:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.780991:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776530.780995:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776530.780996:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776530.780999:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776530.781001:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776530.781004:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776530.781007:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099762560, portal 10 00000100:00000001:3.0:1775776530.781008:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776530.781009:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288720512 : -114841420831104 : ffff978d65e7e680) 00000100:00000040:3.0:1775776530.781014:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781018:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776530.781019:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776530.781020:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776530.781022:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099762560, offset 0 00000400:00000200:3.0:1775776530.781024:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776530.781026:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.781029:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776530.781030:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776530.781032:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776530.781034:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776530.781036:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776530.781039:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4188:4188:256:4294967295] 192.168.201.139@tcp LPNI seq info [4188:4188:8:4294967295] 00000400:00000200:3.0:1775776530.781043:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.781049:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776530.781052:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.781056:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776530.781059:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776530.781062:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776530.781066:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776530.781068:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775776530.781072:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.781074:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776530.781076:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776530.781079:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776530.781082:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776530.781084:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776530.781087:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776530.781090:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781097:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.781099:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781103:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781104:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776530.781107:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776530.781109:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776530.781111:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776530.781112:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312918533 00000100:00000001:3.0:1775776530.781114:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312919797 00000100:00000001:3.0:1775776530.781116:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312918533 00000100:00000001:3.0:1775776530.781117:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312919797 00000100:00000001:3.0:1775776530.781118:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776530.781120:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781121:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776530.781123:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776530/real 1775776530] req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776530.781134:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776530.781135:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776530.781137:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776530.781139:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776530.781140:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776530.781142:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776530.781145:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776530.781152:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781159:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776530.781160:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781162:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.781164:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776530.781167:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781173:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776530.781177:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781181:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776530.781182:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781183:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.781184:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781185:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776530.781187:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776530.781188:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776530.781190:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776530.781191:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776530.781195:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776530.781202:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762560:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776530.781206:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776530.781209:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781215:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776530.781216:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.781218:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776530.781219:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776530.781221:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776530.781224:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762688:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776530.781227:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776530.781229:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776530.781230:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.781232:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776530.781235:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776530.781237:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776530.781239:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776530.781241:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776530.781242:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776530.781244:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099762688, portal 10 00000100:00000001:3.0:1775776530.781246:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776530.781246:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288725888 : -114841420825728 : ffff978d65e7fb80) 00000100:00000040:3.0:1775776530.781250:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781254:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776530.781255:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776530.781256:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776530.781258:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099762688, offset 0 00000400:00000200:3.0:1775776530.781261:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776530.781264:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.781267:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776530.781269:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776530.781272:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776530.781285:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776530.781288:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776530.781290:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4189:4189:256:4294967295] 192.168.201.139@tcp LPNI seq info [4189:4189:8:4294967295] 00000400:00000200:3.0:1775776530.781294:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.781300:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776530.781305:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.781308:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776530.781312:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776530.781314:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776530.781318:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776530.781321:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775776530.781324:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.781326:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776530.781328:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776530.781331:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776530.781334:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776530.781336:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776530.781338:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776530.781342:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781349:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.781352:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781357:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781359:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776530.781362:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776530.781365:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776530.781367:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776530.781369:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312918534 00000100:00000001:3.0:1775776530.781370:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312919797 00000100:00000001:3.0:1775776530.781372:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312918534 00000100:00000001:3.0:1775776530.781373:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312919797 00000100:00000001:3.0:1775776530.781374:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776530.781376:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781377:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776530.781379:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776530/real 1775776530] req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776530.781384:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776530.781385:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776530.781386:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776530.781387:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776530.781388:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776530.781389:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776530.781391:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776530.781398:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781404:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776530.781405:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781407:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.781408:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776530.781411:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781417:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776530.781421:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781427:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776530.781429:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781431:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.781432:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781433:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776530.781435:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776530.781437:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776530.781440:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776530.781442:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776530.781446:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776530.781452:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762688:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776530.781456:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776530.781459:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781465:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776530.781466:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.781468:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776530.781469:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776530.781470:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776530.781472:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762816:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776530.781474:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776530.781475:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776530.781476:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.781477:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776530.781479:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776530.781480:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776530.781483:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776530.781484:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776530.781487:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776530.781488:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099762816, portal 4 00000100:00000001:3.0:1775776530.781490:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776530.781491:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288719616 : -114841420832000 : ffff978d65e7e300) 00000100:00000040:3.0:1775776530.781494:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781498:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776530.781499:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776530.781501:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776530.781502:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099762816, offset 0 00000400:00000200:3.0:1775776530.781505:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776530.781507:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.781509:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776530.781511:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776530.781512:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776530.781514:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776530.781515:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776530.781517:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4190:4190:256:4294967295] 192.168.201.139@tcp LPNI seq info [4190:4190:8:4294967295] 00000400:00000200:3.0:1775776530.781519:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.781523:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776530.781526:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.781527:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776530.781530:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776530.781531:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776530.781534:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776530.781535:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775776530.781538:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.781539:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776530.781541:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776530.781543:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776530.781544:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776530.781545:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776530.781547:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776530.781550:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781562:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.781564:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781570:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781572:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776530.781574:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776530.781577:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776530.781580:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776530.781581:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312918543 00000100:00000001:3.0:1775776530.781584:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312919797 00000100:00000001:3.0:1775776530.781585:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312918543 00000100:00000001:3.0:1775776530.781588:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312919797 00000100:00000001:3.0:1775776530.781590:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776530.781591:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781593:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776530.781596:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776530/real 1775776530] req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776530.781605:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776530.781606:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776530.781608:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776530.781610:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776530.781612:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776530.781614:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776530.781616:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776530.781624:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781630:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776530.781631:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781633:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.781634:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776530.781636:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781640:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776530.781642:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781646:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776530.781647:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781648:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.781649:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781650:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776530.781651:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776530.781652:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776530.781654:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776530.781655:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776530.781657:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776530.781662:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762816:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776530.781665:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781667:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.781669:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e680 x1862033099762560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781675:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776530.781677:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776530.781679:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776530.781681:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776530.781682:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776530.781684:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776530.781686:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776530.781688:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506000. 02000000:00000001:3.0:1775776530.781690:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776530.781691:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776530.781692:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e680. 00000100:00000001:3.0:1775776530.781694:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776530.781696:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781697:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.781700:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7fb80 x1862033099762688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781706:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776530.781707:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776530.781708:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776530.781710:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776530.781711:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776530.781712:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776530.781713:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776530.781715:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506400. 02000000:00000001:3.0:1775776530.781716:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776530.781717:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776530.781719:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7fb80. 00000100:00000001:3.0:1775776530.781721:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776530.781722:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781723:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.781726:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e300 x1862033099762816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781732:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776530.781733:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776530.781734:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776530.781736:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776530.781737:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776530.781738:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776530.781740:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776530.781741:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:3.0:1775776530.781743:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776530.781744:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776530.781745:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e300. 00000100:00000001:3.0:1775776530.781747:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776530.781748:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781750:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781753:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776530.781754:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781756:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776530.781757:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776530.781759:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776530.781762:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781768:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776530.781770:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.781772:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776530.781773:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776530.781775:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776530.781795:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762944:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776530.781799:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776530.781800:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776530.781802:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776530.781803:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776530.781807:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776530.781808:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776530.781812:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776530.781814:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776530.781817:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776530.781819:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099762944, portal 4 00000100:00000001:3.0:1775776530.781821:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776530.781822:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288710656 : -114841420840960 : ffff978d65e7c000) 00000100:00000040:3.0:1775776530.781827:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781833:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776530.781836:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776530.781838:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776530.781840:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099762944, offset 0 00000400:00000200:3.0:1775776530.781844:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776530.781847:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.781850:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776530.781853:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776530.781856:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776530.781858:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776530.781860:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776530.781863:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4191:4191:256:4294967295] 192.168.201.139@tcp LPNI seq info [4191:4191:8:4294967295] 00000400:00000200:3.0:1775776530.781866:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776530.781872:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776530.781876:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.781880:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776530.781883:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776530.781885:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776530.781889:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776530.781892:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775776530.781895:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776530.781897:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776530.781899:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776530.781902:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776530.781905:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776530.781907:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776530.781909:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776530.781912:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781918:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.781920:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776555 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781926:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781927:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776530.781929:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776530.781932:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776530.781934:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776530.781936:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312918535 00000100:00000001:3.0:1775776530.781938:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312919797 00000100:00000001:3.0:1775776530.781940:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312918535 00000100:00000001:3.0:1775776530.781941:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312919797 00000100:00000001:3.0:1775776530.781943:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776530.781944:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.781946:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776530.781948:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776530/real 1775776530] req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776530.781955:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776530.781956:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776530.781957:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776530.781959:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776530.781960:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776530.781962:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776530.781965:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776530.781972:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781977:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776530.781979:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.781980:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.781982:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776530.781985:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.781991:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776530.781995:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.782001:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776530.782003:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.782005:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776530.782006:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.782008:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776530.782009:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776530.782012:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776530.782015:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776530.782017:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776530.782020:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776530.782028:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099762944:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776530.782031:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.782033:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776530.782036:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c000 x1862033099762944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776555 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776530.782041:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776530.782042:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776530.782044:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776530.782047:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776530.782048:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776530.782049:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776530.782051:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776530.782052:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504400. 02000000:00000001:3.0:1775776530.782054:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776530.782056:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776530.782057:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c000. 00000100:00000001:3.0:1775776530.782059:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776530.782060:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.782062:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776530.782065:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776530.782066:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776530.782068:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776530.782069:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776531.483867:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776531.483874:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776531.483876:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776531.483881:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776531.483885:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776531.483888:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776531.483891:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776531.483893:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776531.483896:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776531.547874:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776531.547880:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776531.547883:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776531.547887:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776531.547892:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776531.547896:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776531.547899:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776531.547903:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776531.547906:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776531.547950:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776531.547953:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776531.547956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776532.059895:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312919797, at 4312919808 00000100:00000010:1.0:1775776532.059911:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000100:00000040:0.0:1775776532.060008:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312919797, at 4312919808 00000100:00000010:0.0:1775776532.060015:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000100:00000001:3.0:1775776532.507891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776532.507896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776532.507898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776532.507901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776532.507904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776532.507907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776532.507909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776532.507910:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776532.507912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776532.571833:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776532.571840:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776532.571842:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776532.571847:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776532.571851:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776532.571855:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776532.571858:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776532.571869:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776532.571872:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776532.572103:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776532.572106:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776532.572109:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776533.531888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776533.531895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776533.531896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776533.531902:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776533.531905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776533.531909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776533.531912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776533.531914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776533.531917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776533.595847:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776533.595852:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776533.595854:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776533.595858:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776533.595861:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776533.595864:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776533.595866:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776533.595868:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776533.595870:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776533.595941:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776533.595944:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776533.595958:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776534.555928:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776534.555936:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776534.555937:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776534.555943:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776534.555948:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776534.555953:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776534.555964:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776534.555967:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776534.555970:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776534.619876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776534.619884:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776534.619886:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776534.619896:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776534.619901:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776534.619906:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776534.619911:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776534.619916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776534.619925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776534.619984:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776534.619988:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776534.619991:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776535.579950:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776535.579959:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776535.579962:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776535.579969:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776535.579973:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776535.579977:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.579980:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776535.579983:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.579986:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.643913:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776535.643923:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776535.643929:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776535.643947:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776535.643955:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776535.643960:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.643963:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776535.643966:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.643970:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776535.644022:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776535.644027:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776535.644032:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776535.899945:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776535.899957:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776535.899961:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776535.899965:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776535.899971:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776535.899981:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18251 00000400:00000200:1.0:1775776535.899993:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776535.900001:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776535.900004:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776535.900008:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900012:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776535.900014:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900016:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776535.900022:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776535.900023:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776535.900027:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776535.900030:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776535.900033:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776535.900035:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776535.900038:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900040:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776535.900042:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900044:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776535.900047:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776535.900050:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900053:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776535.900056:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776535.900060:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776535.900063:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900066:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776535.900068:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776535.900081:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776535.900085:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776535.900090:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776535.900091:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900095:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776535.900099:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776535.900103:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776535.900112:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776535.900120:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776535.900130:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776535.900142:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776535.900147:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776535.900152:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776535.900178:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776535.900181:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776535.900196:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776535.900197:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776535.900198:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776535.900201:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776535.900203:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18251 00000400:00000200:1.0:1775776535.900206:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776535.900209:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776535.900211:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776535.900215:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900217:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776535.900219:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900221:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776535.900223:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776535.900225:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776535.900226:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776535.900229:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776535.900230:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776535.900232:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776535.900234:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1775776535.900236:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900239:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776535.900240:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1775776535.900242:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776535.900244:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775776535.900245:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776535.900249:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776535.900253:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000040:3.0:1775776535.900255:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776535.900256:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776535.900259:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776535.900266:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776535.900269:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1775776535.900270:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900273:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776535.900273:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.900279:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1775776535.900280:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1775776535.900283:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1775776535.900285:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.900288:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900289:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776535.900296:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099763072:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776535.900301:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776535.900306:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1775776535.900307:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776535.900311:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776535.900311:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.900316:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775776535.900317:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776535.900320:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900322:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776535.900323:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:3.0:1775776535.900324:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:1.0:1775776535.900325:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.900330:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776535.900332:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775776535.900336:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.900340:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00100000:1.0:1775776535.900343:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1775776535.900346:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000100:00000001:1.0:1775776535.900354:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1775776535.900356:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000001:1.0:1775776535.900357:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776535.900364:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776535.900369:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099763072, portal 25 00000100:00000001:3.0:1775776535.900374:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776535.900376:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.900377:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287599232 : -114841421952384 : ffff978d65d6ca80) 00000100:00080000:1.0:1775776535.900381:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776535.900386:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:3.0:1775776535.900388:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776535.900389:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776535.900392:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776535.900397:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776535.900403:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18251 00000100:00000001:3.0:1775776535.900405:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1775776535.900410:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775776535.900412:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776535.900415:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776535.900420:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099763072, offset 0 00000400:00000200:1.0:1775776535.900430:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000200:3.0:1775776535.900432:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00080000:1.0:1775776535.900435:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776535.900440:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900443:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775776535.900444:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:1.0:1775776535.900447:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900451:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776535.900454:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:3.0:1775776535.900456:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000020:00000040:1.0:1775776535.900461:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776535.900463:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776535.900465:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776535.900467:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776535.900469:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:3.0:1775776535.900473:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1775776535.900474:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900477:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776535.900479:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900480:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776535.900482:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:3.0:1775776535.900485:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00080000:1.0:1775776535.900487:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900489:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776535.900492:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1775776535.900492:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1775776535.900495:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776535.900498:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.900499:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900501:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776535.900504:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776535.900505:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900507:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776535.900511:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000400:00000200:3.0:1775776535.900513:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4192:4192:256:4294967295] 192.168.201.139@tcp LPNI seq info [4192:4192:8:4294967295] 00000020:00000040:1.0:1775776535.900514:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776535.900520:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776535.900522:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.900523:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:1.0:1775776535.900526:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776535.900529:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776535.900531:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776535.900535:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776535.900541:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.900541:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00100000:1.0:1775776535.900550:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1775776535.900555:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.900564:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000100:00000001:1.0:1775776535.900565:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776535.900569:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776535.900572:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1775776535.900579:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776535.900580:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776535.900584:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.0:1775776535.900586:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00080000:1.0:1775776535.900587:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776535.900589:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776535.900592:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776535.900594:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776535.900598:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18251 00000800:00000200:3.1:1775776535.900598:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000400:00000200:1.0:1775776535.900604:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000800:00000200:3.1:1775776535.900604:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000400:00000200:1.0:1775776535.900606:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776535.900609:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776535.900614:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000100:3.0:1775776535.900615:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900617:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1775776535.900619:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000100:00000040:1.0:1775776535.900620:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900625:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776535.900627:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776535.900628:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000400:00000200:3.0:1775776535.900629:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:1.0:1775776535.900630:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776535.900633:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776535.900637:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000400:00000200:3.0:1775776535.900638:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000020:00000001:1.0:1775776535.900642:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776535.900646:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776535.900646:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000100:00000001:1.0:1775776535.900651:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1775776535.900652:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000040:1.0:1775776535.900655:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900658:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776535.900659:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:1.0:1775776535.900663:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000200:3.0:1775776535.900666:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776535.900668:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900674:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776535.900678:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776535.900682:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776535.900685:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.900685:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775776535.900689:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:3.0:1775776535.900692:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775776535.900693:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776535.900699:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900703:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776535.900709:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000100:00000001:3.0:1775776535.900709:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.900712:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:1.0:1775776535.900713:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776535.900717:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1775776535.900719:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776535.900720:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900725:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776535.900726:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776535.900728:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776535.900733:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776535.900739:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.900740:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00100000:1.0:1775776535.900745:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776535.900745:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312924917 00000100:00000001:1.0:1775776535.900752:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776535.900754:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776535.900754:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503447e0. 00000100:00000040:1.0:1775776535.900760:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776535.900765:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.900765:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1775776535.900769:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776535.900772:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776535.900773:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776535.900774:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776535.900776:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776535.900777:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312924917 00000100:00000010:3.0:1775776535.900820:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503445a0. 00000100:00080000:1.0:1775776535.900824:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18251 00000100:00000001:3.0:1775776535.900825:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000400:00000200:1.0:1775776535.900829:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776535.900831:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776535.900832:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776535.900835:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775776535.900837:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776535.900839:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900841:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776535.900842:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1775776535.900845:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900847:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776535.900848:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776535.900850:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776535.900851:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00100000:3.0:1775776535.900851:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776535/real 1775776535] req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775776535.900854:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776535.900857:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776535.900860:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776535.900866:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900870:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776535.900872:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900874:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776535.900875:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000400:00000001:3.0:1775776535.900876:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1775776535.900877:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776535.900879:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1775776535.900879:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1775776535.900882:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:3.0:1775776535.900882:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 02000000:00000001:1.0:1775776535.900885:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775776535.900886:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 02000000:00000001:1.0:1775776535.900887:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776535.900889:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:1.0:1775776535.900890:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776535.900892:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776535.900894:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.900894:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1775776535.900897:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776535.900900:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000100:00000200:3.0:1775776535.900900:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775776535.900904:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776535.900908:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776535.900910:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776535.900913:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776535.900915:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776535.900918:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775776535.900920:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775776535.900922:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776535.900926:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776535.900933:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776535.900934:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776535.900937:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.900941:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776535.900943:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776535.900944:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776535.900947:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776535.900952:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776535.900953:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776535.900977:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.900982:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776535.900983:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18261) 00000100:00000040:3.0:1775776535.900990:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.900998:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776535.901002:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901004:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.901005:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901008:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776535.901010:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776535.901012:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776535.901026:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776535.901028:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776535.901033:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776535.901040:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099763072:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776535.901045:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901047:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.901050:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ca80 x1862033099763072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901056:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776535.901059:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776535.901062:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776535.901065:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776535.901067:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776535.901069:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776535.901071:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776535.901074:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39800. 02000000:00000001:3.0:1775776535.901078:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776535.901081:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776535.901082:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ca80. 00000100:00000001:3.0:1775776535.901085:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776535.901086:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901089:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901097:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776535.901099:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.901102:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776535.901103:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776535.901105:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776535.901108:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901114:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776535.901115:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.901118:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776535.901119:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776535.901121:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776535.901124:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763200:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776535.901127:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776535.901129:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776535.901130:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.901132:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776535.901134:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776535.901136:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.901139:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776535.901141:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776535.901144:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776535.901147:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099763200, portal 10 00000100:00000001:3.0:1775776535.901149:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776535.901150:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287604608 : -114841421947008 : ffff978d65d6df80) 00000100:00000040:3.0:1775776535.901157:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901164:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776535.901166:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776535.901168:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776535.901170:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099763200, offset 0 00000400:00000200:3.0:1775776535.901174:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776535.901178:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776535.901182:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776535.901197:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776535.901200:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776535.901202:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776535.901205:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776535.901208:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4193:4193:256:4294967295] 192.168.201.139@tcp LPNI seq info [4193:4193:8:4294967295] 00000400:00000200:3.0:1775776535.901212:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776535.901218:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776535.901222:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.901226:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776535.901229:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776535.901232:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775776535.901236:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776535.901238:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775776535.901242:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.901244:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776535.901247:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776535.901250:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776535.901252:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776535.901254:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776535.901256:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776535.901260:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901273:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.901275:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901281:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.901283:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776535.901285:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776535.901289:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776535.901292:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776535.901294:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312923653 00000100:00000001:3.0:1775776535.901297:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312924917 00000100:00000001:3.0:1775776535.901299:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312923653 00000100:00000001:3.0:1775776535.901301:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312924917 00000100:00000001:3.0:1775776535.901303:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776535.901305:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.901306:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776535.901309:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776535/real 1775776535] req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776535.901317:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776535.901318:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776535.901319:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776535.901321:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776535.901323:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776535.901325:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776535.901327:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776535.901335:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901341:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776535.901342:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901344:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.901345:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776535.901348:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901354:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776535.901357:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901364:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776535.901367:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901369:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.901370:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901372:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776535.901374:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776535.901376:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776535.901379:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776535.901381:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776535.901384:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776535.901392:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763200:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776535.901395:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776535.901398:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901404:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776535.901405:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.901407:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776535.901409:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776535.901411:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776535.901414:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763328:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776535.901417:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776535.901418:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776535.901420:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.901422:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776535.901425:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776535.901426:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.901429:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776535.901432:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776535.901435:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776535.901437:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099763328, portal 10 00000100:00000001:3.0:1775776535.901439:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776535.901441:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601920 : -114841421949696 : ffff978d65d6d500) 00000100:00000040:3.0:1775776535.901446:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901452:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776535.901454:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776535.901456:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776535.901459:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099763328, offset 0 00000400:00000200:3.0:1775776535.901462:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776535.901466:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776535.901469:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776535.901472:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776535.901475:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776535.901477:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776535.901479:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776535.901482:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4194:4194:256:4294967295] 192.168.201.139@tcp LPNI seq info [4194:4194:8:4294967295] 00000400:00000200:3.0:1775776535.901485:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776535.901491:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776535.901495:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.901498:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776535.901502:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776535.901504:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775776535.901508:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776535.901510:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775776535.901514:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.901516:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776535.901518:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776535.901521:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776535.901523:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776535.901525:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776535.901527:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776535.901530:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901537:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.901540:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901546:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.901547:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776535.901549:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776535.901552:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776535.901555:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776535.901557:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312923654 00000100:00000001:3.0:1775776535.901560:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312924917 00000100:00000001:3.0:1775776535.901562:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312923654 00000100:00000001:3.0:1775776535.901564:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312924917 00000100:00000001:3.0:1775776535.901566:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776535.901568:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.901570:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776535.901572:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776535/real 1775776535] req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776535.901580:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776535.901581:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776535.901583:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776535.901585:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776535.901586:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776535.901588:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776535.901591:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776535.901598:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901604:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776535.901605:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901607:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.901608:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776535.901611:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901618:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776535.901621:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901627:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776535.901630:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901631:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.901632:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901634:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776535.901635:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776535.901637:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776535.901640:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776535.901642:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776535.901645:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776535.901658:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763328:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776535.901661:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776535.901664:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901670:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776535.901671:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.901673:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776535.901675:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776535.901677:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776535.901679:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763456:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776535.901683:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776535.901684:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776535.901686:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.901687:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776535.901690:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776535.901692:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.901695:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776535.901697:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776535.901701:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776535.901703:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099763456, portal 4 00000100:00000001:3.0:1775776535.901705:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776535.901706:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287606400 : -114841421945216 : ffff978d65d6e680) 00000100:00000040:3.0:1775776535.901711:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901716:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776535.901719:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776535.901721:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776535.901723:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099763456, offset 0 00000400:00000200:3.0:1775776535.901726:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776535.901730:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776535.901733:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776535.901735:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776535.901738:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776535.901740:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776535.901742:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776535.901745:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4195:4195:256:4294967295] 192.168.201.139@tcp LPNI seq info [4195:4195:8:4294967295] 00000400:00000200:3.0:1775776535.901749:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776535.901754:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776535.901758:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.901761:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776535.901764:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776535.901766:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775776535.901769:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776535.901772:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775776535.901775:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.901777:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776535.901800:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776535.901804:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776535.901806:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776535.901808:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776535.901810:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776535.901813:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901820:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.901823:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901829:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.901831:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776535.901833:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776535.901836:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776535.901839:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776535.901841:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312923664 00000100:00000001:3.0:1775776535.901843:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312924917 00000100:00000001:3.0:1775776535.901845:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312923664 00000100:00000001:3.0:1775776535.901847:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312924917 00000100:00000001:3.0:1775776535.901849:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776535.901851:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.901852:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776535.901855:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776535/real 1775776535] req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776535.901862:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776535.901863:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776535.901864:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776535.901866:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776535.901868:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776535.901870:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776535.901872:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776535.901881:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901887:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776535.901888:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901890:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.901892:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776535.901894:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901901:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776535.901904:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901911:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776535.901914:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901915:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.901917:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.901918:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776535.901920:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776535.901922:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776535.901925:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776535.901927:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776535.901930:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776535.901937:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763456:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776535.901940:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776535.901942:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901948:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776535.901949:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.901951:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776535.901952:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776535.901954:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776535.901956:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763584:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776535.901959:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776535.901960:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776535.901962:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776535.901963:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776535.901966:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776535.901968:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776535.901970:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776535.901973:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776535.901975:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776535.901977:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099763584, portal 4 00000100:00000001:3.0:1775776535.901979:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776535.901981:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287607296 : -114841421944320 : ffff978d65d6ea00) 00000100:00000040:3.0:1775776535.901986:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.901991:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776535.901993:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776535.901994:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776535.901996:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099763584, offset 0 00000400:00000200:3.0:1775776535.902000:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776535.902003:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776535.902005:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776535.902008:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776535.902010:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776535.902012:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776535.902014:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776535.902016:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4196:4196:256:4294967295] 192.168.201.139@tcp LPNI seq info [4196:4196:8:4294967295] 00000400:00000200:3.0:1775776535.902020:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776535.902024:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776535.902028:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.902030:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776535.902039:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776535.902041:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775776535.902045:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776535.902047:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775776535.902065:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776535.902067:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776535.902069:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776535.902072:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776535.902075:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776535.902076:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776535.902078:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776535.902081:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902093:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.902096:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776560 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902101:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.902102:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776535.902105:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776535.902107:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776535.902110:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776535.902112:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312923655 00000100:00000001:3.0:1775776535.902114:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312924917 00000100:00000001:3.0:1775776535.902116:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312923655 00000100:00000001:3.0:1775776535.902118:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312924917 00000100:00000001:3.0:1775776535.902120:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776535.902121:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.902123:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776535.902126:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776535/real 1775776535] req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776535.902133:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776535.902134:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776535.902136:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776535.902138:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776535.902139:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776535.902142:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776535.902145:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776535.902152:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902157:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776535.902159:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902160:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.902161:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776535.902164:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902169:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776535.902172:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902178:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776535.902180:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902181:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776535.902182:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902196:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776535.902197:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776535.902199:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776535.902202:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776535.902203:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776535.902207:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776535.902213:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763584:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776535.902216:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902218:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.902221:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6df80 x1862033099763200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902226:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776535.902228:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776535.902230:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776535.902232:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776535.902234:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776535.902235:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776535.902237:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776535.902238:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38400. 02000000:00000001:3.0:1775776535.902241:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776535.902242:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776535.902244:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6df80. 00000100:00000001:3.0:1775776535.902246:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776535.902248:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902249:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.902252:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d500 x1862033099763328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902258:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776535.902259:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776535.902261:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776535.902263:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776535.902264:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776535.902265:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776535.902267:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776535.902268:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38c00. 02000000:00000001:3.0:1775776535.902270:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776535.902271:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776535.902273:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d500. 00000100:00000001:3.0:1775776535.902275:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776535.902276:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902278:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.902281:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6e680 x1862033099763456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902286:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776535.902287:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776535.902288:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776535.902290:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776535.902291:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776535.902292:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776535.902294:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776535.902295:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39400. 02000000:00000001:3.0:1775776535.902297:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776535.902298:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776535.902299:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6e680. 00000100:00000001:3.0:1775776535.902301:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776535.902302:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902303:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776535.902306:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ea00 x1862033099763584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776560 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776535.902311:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776535.902312:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776535.902313:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776535.902314:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776535.902315:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776535.902316:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776535.902317:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776535.902318:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a400. 02000000:00000001:3.0:1775776535.902320:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776535.902321:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776535.902322:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ea00. 00000100:00000001:3.0:1775776535.902324:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776535.902325:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902326:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776535.902331:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776535.902332:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776535.902333:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776535.902334:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776536.603858:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776536.603865:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776536.603867:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776536.603872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776536.603875:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776536.603879:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776536.603881:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776536.603883:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776536.603890:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776536.667914:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776536.667920:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776536.667922:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776536.667935:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776536.667939:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776536.667943:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776536.667946:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776536.667948:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776536.667951:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776536.668041:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776536.668045:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776536.668048:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776537.179882:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312924917, at 4312924928 00000100:00000040:1.0:1775776537.179882:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312924917, at 4312924928 00000100:00000010:1.0:1775776537.179898:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503445a0. 00000100:00000010:0.0:1775776537.179907:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503447e0. 00000100:00000001:3.0:1775776537.627889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776537.627896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776537.627898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776537.627904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776537.627908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776537.627912:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776537.627914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776537.627915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776537.627917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776537.692008:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776537.692012:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776537.692015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776537.693278:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776537.693282:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776537.693284:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776537.693288:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776537.693291:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776537.693294:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776537.693297:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776537.693299:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776537.693302:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.651888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776538.651895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776538.651897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776538.651902:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776538.651905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776538.651909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776538.651911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776538.651914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776538.651916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776538.716003:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776538.716006:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776538.716008:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776538.716040:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776538.716045:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776538.716046:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776538.716050:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776538.716053:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776538.716056:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776538.716058:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776538.716060:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776538.716063:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776539.676242:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776539.676248:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776539.676250:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776539.676256:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776539.676259:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776539.676263:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776539.676266:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776539.676268:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776539.676271:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776539.739896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776539.739903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776539.739907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776539.739913:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776539.739919:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776539.739923:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776539.739927:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776539.739932:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776539.739936:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776539.740040:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776539.740043:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776539.740045:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776540.699860:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776540.699867:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776540.699875:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776540.699880:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776540.699883:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776540.699887:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776540.699889:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776540.699892:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776540.699894:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776540.763889:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776540.763896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776540.763899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776540.763905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776540.763910:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776540.763914:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776540.763918:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776540.763922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776540.763926:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776540.763984:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776540.763987:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776540.763989:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776541.019910:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776541.019921:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776541.019925:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776541.019929:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776541.019933:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776541.019940:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18256 00000400:00000200:1.0:1775776541.019950:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776541.019956:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776541.019960:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776541.019963:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.019966:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776541.019968:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.019971:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776541.019974:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776541.019976:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776541.019979:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776541.019981:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776541.019984:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776541.019986:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776541.019989:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.019991:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776541.019992:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.019994:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776541.019997:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776541.020000:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020011:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776541.020018:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776541.020022:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776541.020025:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.020027:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776541.020029:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776541.020038:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776541.020042:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776541.020046:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776541.020048:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.020051:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776541.020053:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776541.020058:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776541.020066:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776541.020073:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776541.020081:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776541.020091:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776541.020109:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776541.020114:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776541.020138:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776541.020141:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776541.020144:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776541.020146:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776541.020147:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776541.020150:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776541.020152:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18256 00000400:00000200:1.0:1775776541.020156:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776541.020159:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776541.020162:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776541.020165:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020167:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776541.020170:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020172:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776541.020174:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776541.020176:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776541.020178:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776541.020178:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1775776541.020180:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776541.020184:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776541.020184:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775776541.020188:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776541.020188:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776541.020193:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020197:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776541.020198:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776541.020201:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020206:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776541.020211:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000001:3.0:1775776541.020213:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776541.020216:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776541.020217:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 02000000:00000001:3.0:1775776541.020219:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775776541.020220:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.020222:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776541.020224:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776541.020225:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776541.020229:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099763712:192.168.201.139@tcp:250:kworker.0 02000000:00000001:1.0:1775776541.020230:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776541.020232:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776541.020236:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.020236:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1775776541.020238:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:3.0:1775776541.020239:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.020242:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1775776541.020244:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 02000000:00000010:3.0:1775776541.020246:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 00000008:00000001:1.0:1775776541.020248:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.020248:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.020253:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776541.020257:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000010:1.0:1775776541.020259:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776541.020262:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 00000400:00000010:3.0:1775776541.020262:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 02000000:00000001:1.0:1775776541.020268:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776541.020270:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.020271:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776541.020273:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1775776541.020274:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 02000000:00000001:1.0:1775776541.020275:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776541.020281:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000200:3.0:1775776541.020283:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099763712, portal 25 00000100:00000001:1.0:1775776541.020286:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020287:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776541.020290:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287607296 : -114841421944320 : ffff978d65d6ea00) 00000100:00100000:1.0:1775776541.020295:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776541.020303:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776541.020307:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776541.020312:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776541.020314:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.020333:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776541.020340:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776541.020344:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000001:1.0:1775776541.020348:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775776541.020350:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099763712, offset 0 00000100:00080000:1.0:1775776541.020352:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776541.020356:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776541.020360:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1775776541.020360:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00080000:1.0:1775776541.020363:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776541.020367:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775776541.020370:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:1.0:1775776541.020372:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18256 00000400:00000200:1.0:1775776541.020378:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.020381:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:1.0:1775776541.020385:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000200:3.0:1775776541.020388:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1775776541.020389:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776541.020394:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776541.020396:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000001:1.0:1775776541.020398:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775776541.020402:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1775776541.020403:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020407:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:3.0:1775776541.020409:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000020:00000001:1.0:1775776541.020411:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776541.020413:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776541.020417:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1775776541.020417:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4197:4197:256:4294967295] 192.168.201.139@tcp LPNI seq info [4197:4197:8:4294967295] 00000020:00000040:1.0:1775776541.020420:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776541.020425:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:3.0:1775776541.020429:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:1.0:1775776541.020430:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776541.020436:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020439:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776541.020443:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020448:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:3.0:1775776541.020448:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000020:00000040:1.0:1775776541.020453:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000800:00000200:3.0:1775776541.020458:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00080000:1.0:1775776541.020461:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020464:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775776541.020464:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 02000000:00000001:1.0:1775776541.020468:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776541.020471:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1775776541.020472:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775776541.020477:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775776541.020477:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:1.0:1775776541.020482:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:3.1:1775776541.020484:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00100000:1.0:1775776541.020486:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:3.1:1775776541.020487:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000008:00000001:1.0:1775776541.020493:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1775776541.020493:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020497:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775776541.020498:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776541.020504:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:1.0:1775776541.020505:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776541.020509:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000400:00000200:3.0:1775776541.020510:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1775776541.020515:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1775776541.020516:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 02000000:00000001:1.0:1775776541.020517:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776541.020518:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:1.0:1775776541.020522:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1775776541.020523:0:301888:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1775776541.020524:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776541.020527:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775776541.020528:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775776541.020533:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776541.020539:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020545:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1775776541.020547:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1775776541.020552:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776541.020564:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776541.020569:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020570:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020574:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1775776541.020576:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.020581:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.020589:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776541.020589:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776541.020593:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312930037 00000100:00080000:1.0:1775776541.020594:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776541.020598:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776541.020599:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776541.020601:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776541.020603:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776541.020607:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18256 00000100:00000010:3.0:1775776541.020611:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503442a0. 00000100:00000001:3.0:1775776541.020619:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000400:00000200:1.0:1775776541.020623:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776541.020625:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312930037 00000400:00000200:1.0:1775776541.020629:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000010:3.0:1775776541.020631:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503441e0. 00000100:00080000:1.0:1775776541.020632:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775776541.020635:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000040:1.0:1775776541.020638:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776541.020641:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776541.020642:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776541.020646:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776541.020647:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020652:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776541.020652:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1775776541.020656:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000100:00100000:3.0:1775776541.020658:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776541/real 1775776541] req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775776541.020659:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776541.020662:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776541.020665:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776541.020670:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776541.020675:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776541.020683:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000001:3.0:1775776541.020683:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776541.020686:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775776541.020687:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1775776541.020688:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000100:00000040:1.0:1775776541.020692:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776541.020692:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776541.020695:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:1.0:1775776541.020696:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776541.020698:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1775776541.020702:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000200:3.0:1775776541.020703:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776541.020709:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020714:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775776541.020716:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776541.020718:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776541.020723:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776541.020723:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776541.020725:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775776541.020728:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020728:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.020731:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776541.020732:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:3.0:1775776541.020735:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775776541.020737:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776541.020743:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.020748:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020749:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776541.020754:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775776541.020755:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776541.020760:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00000001:3.0:1775776541.020765:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1775776541.020766:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776541.020769:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020769:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776541.020772:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776541.020773:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776541.020775:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776541.020800:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776541.020804:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776541.020810:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776541.020818:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776541.020820:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776541.020822:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020823:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.020826:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000040:1.0:1775776541.020828:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.020829:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776541.020835:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776541.020844:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:1.0:1775776541.020845:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020846:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1775776541.020849:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776541.020854:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776541.020855:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776541.020858:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776541.020861:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776541.020866:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776541.020871:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18256 00000100:00100000:3.0:1775776541.020875:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099763712:192.168.201.139@tcp:250:kworker.0 00000400:00000200:1.0:1775776541.020878:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776541.020884:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:3.0:1775776541.020885:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775776541.020887:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775776541.020888:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776541.020892:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000040:3.0:1775776541.020895:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ea00 x1862033099763712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776541.020896:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776541.020901:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776541.020906:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776541.020907:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1775776541.020911:0:313512:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:3.0:1775776541.020911:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:1.0:1775776541.020914:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 02000000:00000010:3.0:1775776541.020915:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 00000020:00000001:1.0:1775776541.020918:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1775776541.020920:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:1.0:1775776541.020922:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776541.020922:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776541.020924:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:1.0:1775776541.020927:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776541.020927:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776541.020930:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b800. 00000020:00000001:1.0:1775776541.020931:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 02000000:00000001:3.0:1775776541.020934:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1775776541.020937:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000001:3.0:1775776541.020937:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776541.020939:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ea00. 00000100:00000001:1.0:1775776541.020941:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775776541.020943:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776541.020945:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776541.020946:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776541.020949:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776541.020950:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776541.020954:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776541.020957:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776541.020958:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776541.020959:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776541.020962:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775776541.020962:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776541.020967:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776541.020967:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776541.020970:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1775776541.020971:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.020972:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775776541.020974:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776541.020978:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000040:3.0:1775776541.020979:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775776541.020982:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.020986:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776541.020993:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000100:00000001:3.0:1775776541.020994:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1775776541.020997:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:3.0:1775776541.020997:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021001:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776541.021002:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776541.021004:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776541.021005:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021008:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.021009:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776541.021012:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1775776541.021013:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763840:192.168.201.139@tcp:38:kworker.0 02000000:00000001:1.0:1775776541.021015:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776541.021020:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:3.0:1775776541.021021:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1775776541.021025:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021025:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776541.021027:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021030:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1775776541.021033:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:3.0:1775776541.021035:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776541.021040:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776541.021047:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1775776541.021048:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:1.0:1775776541.021052:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776541.021053:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000100:00000040:1.0:1775776541.021058:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1775776541.021059:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776541.021066:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099763840, portal 10 00000100:00000001:3.0:1775776541.021071:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776541.021073:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287606400 : -114841421945216 : ffff978d65d6e680) 00000100:00000001:1.0:1775776541.021074:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776541.021082:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18266) 00000100:00000040:3.0:1775776541.021084:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021091:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776541.021093:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776541.021106:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776541.021109:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099763840, offset 0 00000400:00000200:3.0:1775776541.021115:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776541.021123:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.021127:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776541.021130:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776541.021132:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776541.021135:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776541.021137:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776541.021140:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4198:4198:256:4294967295] 192.168.201.139@tcp LPNI seq info [4198:4198:8:4294967295] 00000400:00000200:3.0:1775776541.021144:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.021150:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776541.021154:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776541.021158:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776541.021162:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776541.021164:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775776541.021167:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776541.021168:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1775776541.021170:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776541.021171:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776541.021173:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776541.021176:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776541.021178:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776541.021179:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776541.021180:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776541.021183:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021190:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776541.021192:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021198:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021199:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776541.021202:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776541.021205:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776541.021208:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776541.021210:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312928773 00000100:00000001:3.0:1775776541.021213:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312930037 00000100:00000001:3.0:1775776541.021215:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312928773 00000100:00000001:3.0:1775776541.021217:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312930037 00000100:00000001:3.0:1775776541.021219:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776541.021221:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021223:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776541.021226:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776541/real 1775776541] req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776541.021234:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776541.021235:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776541.021238:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776541.021240:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776541.021241:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776541.021243:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776541.021245:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776541.021251:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021255:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776541.021255:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021257:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.021257:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776541.021259:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021267:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776541.021269:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021274:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776541.021275:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021276:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.021277:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021278:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776541.021279:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776541.021280:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776541.021282:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776541.021283:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776541.021285:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776541.021290:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763840:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776541.021293:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776541.021296:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021302:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776541.021303:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021305:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776541.021307:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776541.021309:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776541.021312:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763968:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776541.021315:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776541.021317:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776541.021318:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021320:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776541.021323:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776541.021325:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776541.021328:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776541.021330:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776541.021333:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776541.021336:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099763968, portal 10 00000100:00000001:3.0:1775776541.021337:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776541.021338:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601920 : -114841421949696 : ffff978d65d6d500) 00000100:00000040:3.0:1775776541.021344:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021349:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776541.021351:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776541.021353:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776541.021355:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099763968, offset 0 00000400:00000200:3.0:1775776541.021359:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776541.021363:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.021366:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776541.021368:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776541.021371:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776541.021373:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776541.021376:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776541.021378:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4199:4199:256:4294967295] 192.168.201.139@tcp LPNI seq info [4199:4199:8:4294967295] 00000400:00000200:3.0:1775776541.021382:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.021387:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776541.021391:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776541.021395:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776541.021398:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776541.021400:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775776541.021404:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776541.021405:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1775776541.021409:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776541.021411:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776541.021413:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776541.021416:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776541.021419:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776541.021420:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776541.021422:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776541.021426:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021432:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776541.021435:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021441:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021442:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776541.021445:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776541.021448:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776541.021451:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776541.021453:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312928774 00000100:00000001:3.0:1775776541.021456:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312930037 00000100:00000001:3.0:1775776541.021458:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312928774 00000100:00000001:3.0:1775776541.021460:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312930037 00000100:00000001:3.0:1775776541.021462:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776541.021463:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021465:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776541.021469:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776541/real 1775776541] req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776541.021475:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776541.021475:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776541.021476:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776541.021478:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776541.021479:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776541.021480:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776541.021482:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776541.021486:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021490:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776541.021490:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021491:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.021492:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776541.021494:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021497:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776541.021499:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021503:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776541.021504:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021505:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.021506:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021507:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776541.021509:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776541.021510:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776541.021512:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776541.021513:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776541.021515:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776541.021519:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099763968:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776541.021521:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776541.021523:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021527:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776541.021528:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021530:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776541.021531:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776541.021533:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776541.021535:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099764096:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776541.021538:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776541.021540:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776541.021541:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021543:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776541.021545:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776541.021547:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776541.021551:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776541.021553:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776541.021555:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776541.021558:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099764096, portal 4 00000100:00000001:3.0:1775776541.021560:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776541.021561:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287604608 : -114841421947008 : ffff978d65d6df80) 00000100:00000040:3.0:1775776541.021566:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021574:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776541.021576:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776541.021578:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776541.021580:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099764096, offset 0 00000400:00000200:3.0:1775776541.021584:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776541.021588:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.021591:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776541.021594:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776541.021596:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776541.021598:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776541.021600:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776541.021602:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4200:4200:256:4294967295] 192.168.201.139@tcp LPNI seq info [4200:4200:8:4294967295] 00000400:00000200:3.0:1775776541.021606:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.021611:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776541.021616:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776541.021619:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776541.021623:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776541.021625:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775776541.021629:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776541.021631:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1775776541.021634:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776541.021636:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776541.021639:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776541.021642:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776541.021644:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776541.021646:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776541.021649:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776541.021652:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021656:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776541.021658:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021666:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021667:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776541.021669:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776541.021671:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776541.021673:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776541.021674:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312928783 00000100:00000001:3.0:1775776541.021676:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312930037 00000100:00000001:3.0:1775776541.021677:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312928783 00000100:00000001:3.0:1775776541.021678:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312930037 00000100:00000001:3.0:1775776541.021679:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776541.021680:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021681:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776541.021683:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776541/real 1775776541] req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776541.021688:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776541.021689:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776541.021690:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776541.021691:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776541.021692:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776541.021693:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776541.021695:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776541.021699:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021702:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776541.021703:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021704:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.021705:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776541.021706:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021711:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776541.021713:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021717:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776541.021718:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021719:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.021719:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021720:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776541.021721:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776541.021723:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776541.021724:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776541.021725:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776541.021727:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776541.021731:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099764096:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776541.021733:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021734:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776541.021735:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6e680 x1862033099763840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021739:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776541.021740:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776541.021741:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776541.021742:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776541.021743:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776541.021744:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776541.021746:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776541.021747:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3bc00. 02000000:00000001:3.0:1775776541.021748:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776541.021749:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776541.021750:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6e680. 00000100:00000001:3.0:1775776541.021752:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776541.021753:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021753:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776541.021755:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d500 x1862033099763968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021759:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776541.021759:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776541.021760:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776541.021761:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776541.021762:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776541.021762:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776541.021763:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776541.021764:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39000. 02000000:00000001:3.0:1775776541.021765:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776541.021766:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776541.021766:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d500. 00000100:00000001:3.0:1775776541.021767:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776541.021768:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021769:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776541.021770:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6df80 x1862033099764096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021774:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776541.021775:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776541.021776:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776541.021796:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776541.021797:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776541.021798:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776541.021799:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776541.021800:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3ac00. 02000000:00000001:3.0:1775776541.021802:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776541.021803:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776541.021804:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6df80. 00000100:00000001:3.0:1775776541.021806:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776541.021808:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021810:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.021813:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776541.021814:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021816:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776541.021818:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776541.021819:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776541.021823:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021829:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776541.021830:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021832:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776541.021833:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776541.021835:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776541.021838:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099764224:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776541.021840:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776541.021841:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776541.021842:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776541.021844:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776541.021846:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776541.021848:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776541.021851:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776541.021853:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776541.021856:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776541.021859:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099764224, portal 4 00000100:00000001:3.0:1775776541.021861:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776541.021862:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287599232 : -114841421952384 : ffff978d65d6ca80) 00000100:00000040:3.0:1775776541.021867:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021874:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776541.021876:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776541.021878:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776541.021880:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099764224, offset 0 00000400:00000200:3.0:1775776541.021884:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776541.021888:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.021891:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776541.021894:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776541.021896:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776541.021898:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776541.021900:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776541.021902:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4201:4201:256:4294967295] 192.168.201.139@tcp LPNI seq info [4201:4201:8:4294967295] 00000400:00000200:3.0:1775776541.021905:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776541.021909:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776541.021912:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776541.021914:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776541.021916:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776541.021918:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775776541.021920:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776541.021922:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1775776541.021924:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776541.021925:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776541.021927:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776541.021929:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776541.021931:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776541.021932:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776541.021933:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776541.021935:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021940:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776541.021941:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776566 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.021948:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021949:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776541.021951:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776541.021954:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776541.021957:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776541.021959:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312928775 00000100:00000001:3.0:1775776541.021961:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312930037 00000100:00000001:3.0:1775776541.021962:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312928775 00000100:00000001:3.0:1775776541.021964:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312930037 00000100:00000001:3.0:1775776541.021966:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776541.021969:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.021970:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776541.021974:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776541/real 1775776541] req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776541.021981:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776541.021983:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776541.021984:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776541.021987:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776541.021988:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776541.021990:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776541.021993:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776541.022000:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.022006:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776541.022007:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.022008:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.022009:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776541.022011:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.022017:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776541.022020:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.022026:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776541.022029:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.022030:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776541.022031:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.022032:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776541.022034:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776541.022036:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776541.022040:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776541.022041:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776541.022045:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776541.022053:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099764224:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776541.022056:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.022058:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776541.022061:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ca80 x1862033099764224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776566 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776541.022067:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776541.022068:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776541.022070:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776541.022072:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776541.022073:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776541.022074:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776541.022076:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776541.022077:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38000. 02000000:00000001:3.0:1775776541.022078:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776541.022079:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776541.022080:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ca80. 00000100:00000001:3.0:1775776541.022081:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776541.022082:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.022083:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.022085:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776541.022086:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.022088:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776541.022089:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776541.723881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776541.723887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776541.723888:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776541.723893:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776541.723895:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776541.723898:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.723900:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776541.723903:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776541.723904:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776541.787888:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776541.787894:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776541.787896:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776541.787903:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776541.787907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776541.787911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776541.787913:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776541.787915:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776541.787919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776541.787985:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776541.787990:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776541.787993:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776542.299892:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312930037, at 4312930048 00000100:00000040:1.0:1775776542.299892:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312930037, at 4312930048 00000100:00000010:0.0:1775776542.299908:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503442a0. 00000100:00000010:1.0:1775776542.299911:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000100:00000001:3.0:1775776542.747874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776542.747879:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776542.747881:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776542.747885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776542.747888:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776542.747891:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776542.747892:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776542.747894:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776542.747896:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776542.811876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776542.811882:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776542.811885:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776542.811890:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776542.811896:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776542.811901:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776542.811905:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776542.811909:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776542.811912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776542.811955:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776542.811958:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776542.811960:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776543.771879:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776543.771886:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776543.771887:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776543.771891:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776543.771894:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776543.771897:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776543.771899:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776543.771900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776543.771902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776543.835868:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776543.835873:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776543.835875:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776543.835881:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776543.835884:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776543.835888:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776543.835892:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776543.835895:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776543.835896:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776543.835971:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776543.835975:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776543.835977:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776544.795896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776544.795903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776544.795905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776544.795910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776544.795913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776544.795916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776544.795919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776544.795921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776544.795923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776544.859886:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776544.859893:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776544.859894:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776544.859898:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776544.859901:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776544.859904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776544.859906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776544.859908:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776544.859910:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776544.859963:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776544.859967:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776544.859970:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776545.819885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776545.819891:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776545.819893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776545.819898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776545.819901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776545.819905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776545.819908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776545.819911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776545.819913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776545.883884:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776545.883889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776545.883890:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776545.883895:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776545.883911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776545.883915:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776545.883917:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776545.883918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776545.883920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776545.884093:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776545.884097:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776545.884101:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776546.139916:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776546.139925:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776546.139927:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776546.139930:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776546.139934:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776546.139941:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18261 00000400:00000200:3.0:1775776546.139950:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776546.139956:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776546.139959:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776546.139962:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.139965:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.139967:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.139969:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776546.139973:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776546.139976:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776546.139978:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776546.139981:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776546.139983:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776546.139985:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776546.139988:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.139990:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.139991:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.139993:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776546.139996:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776546.140000:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140002:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776546.140015:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776546.140018:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776546.140020:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140023:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776546.140025:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776546.140035:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776546.140039:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776546.140042:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776546.140045:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140048:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776546.140050:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776546.140054:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776546.140061:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776546.140067:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776546.140074:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099764352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776546.140081:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776546.140084:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776546.140087:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099764352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776546.140105:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776546.140108:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776546.140110:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776546.140111:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776546.140112:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776546.140114:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776546.140115:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18261 00000400:00000200:3.0:1775776546.140118:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776546.140120:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776546.140122:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776546.140124:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140125:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.140128:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140131:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776546.140133:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776546.140134:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776546.140136:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776546.140138:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776546.140142:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776546.140145:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776546.140151:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140155:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.140160:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140165:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776546.140169:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776546.140174:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140186:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776546.140188:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776546.140193:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776546.140197:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140200:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776546.140204:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776546.140209:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140213:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776546.140218:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776546.140222:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776546.140224:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776546.140227:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140229:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776546.140231:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776546.140233:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776546.140241:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775776546.140246:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776546.140267:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099764480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776546.140284:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776546.140287:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776546.140295:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099764480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776546.140314:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776546.140319:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776546.140326:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776546.140329:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776546.140332:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776546.140335:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776546.140339:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18261 00000400:00000200:3.0:1775776546.140349:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776546.140355:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776546.140359:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776546.140364:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140368:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775776546.140377:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:3.0:1775776546.140380:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776546.140381:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1775776546.140385:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776546.140390:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776546.140393:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776546.140396:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776546.140398:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776546.140403:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776546.140407:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776546.140413:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140416:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.140421:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140426:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776546.140432:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776546.140437:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140443:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776546.140447:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776546.140451:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776546.140454:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140456:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776546.140458:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776546.140461:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140463:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776546.140468:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776546.140470:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776546.140472:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776546.140475:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140476:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776546.140478:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776546.140479:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776546.140483:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776546.140486:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776546.140491:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099764608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776546.140500:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776546.140502:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776546.140507:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099764608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776546.140523:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776546.140528:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776546.140534:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776546.140538:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776546.140542:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776546.140547:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776546.140554:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18261 00000400:00000200:3.0:1775776546.140562:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776546.140569:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776546.140574:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776546.140579:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140584:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.140590:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140595:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776546.140600:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776546.140602:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776546.140605:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776546.140609:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776546.140613:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776546.140618:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776546.140624:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140626:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.140630:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140632:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776546.140634:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776546.140639:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140641:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776546.140642:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776546.140645:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776546.140648:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140650:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776546.140653:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776546.140658:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140662:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776546.140675:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776546.140679:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776546.140684:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776546.140687:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140691:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776546.140694:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776546.140697:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776546.140702:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776546.140707:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776546.140716:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099764736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776546.140736:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776546.140741:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776546.140748:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099764736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776546.140757:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776546.140760:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776546.140766:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776546.140768:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776546.140771:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776546.140803:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776546.140813:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18261 00000400:00000200:3.0:1775776546.140819:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776546.140825:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776546.140829:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776546.140834:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140838:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.140842:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140846:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776546.140851:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776546.140853:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776546.140857:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776546.140860:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776546.140862:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776546.140866:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776546.140872:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140877:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776546.140880:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140883:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776546.140888:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776546.140891:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776546.140894:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776546.140896:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776546.140899:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776546.140901:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140903:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776546.140906:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776546.140908:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140910:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776546.140915:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776546.140918:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776546.140921:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776546.140922:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776546.140924:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776546.140926:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776546.140928:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776546.140933:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776546.140937:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776546.140944:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099764864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776546.140954:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776546.140957:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776546.140962:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099764864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776546.140975:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776546.140979:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18271) 00000800:00000200:2.1:1775776546.141034:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776546.141037:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776546.141227:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776546.141229:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776546.141439:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776546.141441:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776546.141654:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776546.141655:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1775776546.843871:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776546.843877:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776546.843879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776546.843884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776546.843888:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776546.843891:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776546.843894:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776546.843896:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776546.843898:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776546.907890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776546.907895:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776546.907897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776546.907902:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776546.907905:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776546.907908:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776546.907911:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776546.907913:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776546.907916:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776546.907990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776546.908013:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776546.908015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776547.419914:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312935157, at 4312935168 00000100:00000040:1.0:1775776547.419914:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312935157, at 4312935168 00000100:00000010:0.0:1775776547.419933:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000010:1.0:1775776547.419935:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000001:3.0:1775776547.868000:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776547.868007:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776547.868009:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776547.868014:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776547.868017:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776547.868021:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776547.868023:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776547.868026:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776547.868028:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776547.931921:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776547.931928:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776547.931931:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776547.931938:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776547.931946:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776547.931951:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776547.931955:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776547.931958:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776547.931962:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776547.932046:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776547.932049:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776547.932052:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776548.891909:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776548.891916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776548.891918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776548.891923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776548.891927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776548.891930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776548.891933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776548.891935:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776548.891937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776548.955937:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776548.955946:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776548.955949:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776548.955955:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776548.955969:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776548.955974:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776548.955977:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776548.955981:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776548.955985:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776548.956006:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776548.956011:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776548.956016:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776549.915893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776549.915899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776549.915900:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776549.915904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776549.915906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776549.915909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776549.915911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776549.915912:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776549.915914:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776549.979859:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776549.979866:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776549.979869:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776549.979874:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776549.979879:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776549.979883:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776549.979889:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776549.979893:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776549.979898:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776549.979979:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776549.979982:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776549.979985:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776550.939854:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776550.939860:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776550.939861:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776550.939865:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776550.939867:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776550.939870:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776550.939872:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776550.939873:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776550.939875:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776551.003895:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776551.003899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776551.003901:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776551.003904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776551.003907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776551.003910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776551.003913:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776551.003915:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776551.003918:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776551.003963:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776551.003966:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776551.003969:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776551.259876:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776551.259884:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776551.259886:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776551.259889:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776551.259892:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776551.259897:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18266 00000400:00000200:3.0:1775776551.259904:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776551.259908:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776551.259910:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776551.259912:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.259915:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.259945:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.259948:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776551.259951:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776551.259953:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776551.259956:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776551.259958:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776551.259960:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776551.259963:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776551.259966:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.259968:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.259970:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.259972:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776551.259974:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776551.259977:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776551.259980:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776551.259982:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776551.259985:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776551.259987:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.259988:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776551.259990:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776551.259997:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775776551.260000:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776551.260002:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776551.260003:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260005:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776551.260007:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776551.260010:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776551.260015:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776551.260020:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776551.260027:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099764992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776551.260034:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776551.260038:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776551.260041:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099764992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776551.260057:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776551.260059:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776551.260061:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776551.260062:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776551.260063:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776551.260065:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776551.260067:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18266 00000400:00000200:3.0:1775776551.260070:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776551.260071:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776551.260073:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776551.260074:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260076:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.260077:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260079:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776551.260080:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776551.260081:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776551.260083:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776551.260084:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776551.260085:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776551.260086:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776551.260088:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260089:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.260090:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260092:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776551.260095:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776551.260098:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260103:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776551.260106:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776551.260112:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776551.260117:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260121:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776551.260127:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776551.260132:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260137:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776551.260144:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000020:00000040:3.0:1775776551.260149:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776551.260154:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776551.260156:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260157:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776551.260158:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776551.260162:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776551.260169:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776551.260173:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776551.260177:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099765120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776551.260186:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776551.260189:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776551.260194:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099765120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776551.260202:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776551.260205:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776551.260208:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776551.260211:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776551.260215:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776551.260219:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776551.260225:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18266 00000400:00000200:3.0:1775776551.260234:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776551.260241:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776551.260245:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776551.260250:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260255:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.260259:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260264:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776551.260266:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776551.260267:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776551.260269:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776551.260272:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776551.260275:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776551.260279:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776551.260284:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260288:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.260292:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260294:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776551.260297:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776551.260302:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260305:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776551.260307:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776551.260310:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776551.260313:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260315:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776551.260316:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776551.260318:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260319:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776551.260323:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776551.260327:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776551.260330:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776551.260332:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260334:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776551.260335:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776551.260337:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776551.260341:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776551.260344:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776551.260348:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099765248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776551.260355:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776551.260357:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776551.260362:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099765248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776551.260373:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776551.260373:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:3.0:1775776551.260376:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776551.260378:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776551.260378:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776551.260379:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776551.260381:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776551.260383:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18266 00000800:00000200:2.1:1775776551.260388:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000400:00000200:3.0:1775776551.260395:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776551.260401:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776551.260403:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776551.260409:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260413:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.260417:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260422:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776551.260426:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776551.260429:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776551.260432:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776551.260435:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776551.260440:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776551.260445:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776551.260451:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260455:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.260459:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260462:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776551.260464:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776551.260467:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260470:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776551.260472:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776551.260475:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776551.260478:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260480:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776551.260483:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776551.260485:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260486:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776551.260489:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776551.260491:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776551.260493:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776551.260496:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260499:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776551.260502:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776551.260504:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776551.260510:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776551.260516:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776551.260526:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099765376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776551.260543:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776551.260546:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776551.260554:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099765376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776551.260566:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776551.260569:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776551.260573:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776551.260576:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776551.260578:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776551.260581:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776551.260585:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18266 00000400:00000200:3.0:1775776551.260593:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776551.260598:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776551.260600:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776551.260604:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260606:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.260609:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260613:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776551.260616:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776551.260617:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776551.260618:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776551.260619:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776551.260620:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776551.260622:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776551.260623:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260624:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776551.260625:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260626:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776551.260628:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776551.260629:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776551.260630:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776551.260631:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776551.260641:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776551.260643:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260647:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776551.260650:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776551.260656:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260660:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776551.260665:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776551.260668:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776551.260672:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776551.260674:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776551.260676:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776551.260677:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776551.260680:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776551.260685:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776551.260691:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776551.260700:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099765504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776551.260724:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776551.260727:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776551.260734:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099765504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776551.260745:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776551.260750:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18276) 00000800:00000200:2.1:1775776551.260876:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776551.260877:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776551.261081:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776551.261083:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776551.261269:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776551.261270:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776551.261463:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776551.261465:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1775776551.963896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776551.963902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776551.963914:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776551.963918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776551.963922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776551.963926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776551.963929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776551.963931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776551.963933:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776552.027862:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776552.027868:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776552.027872:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776552.027877:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776552.027882:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776552.027888:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776552.027892:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776552.027896:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776552.027900:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776552.027983:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776552.027986:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776552.027989:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776552.539856:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312940277, at 4312940288 00000100:00000040:1.0:1775776552.539856:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312940277, at 4312940288 00000100:00000010:1.0:1775776552.539871:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000010:0.0:1775776552.539877:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:3.0:1775776552.987857:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776552.987865:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776552.987866:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776552.987872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776552.987874:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776552.987877:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776552.987879:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776552.987881:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776552.987883:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776553.051880:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776553.051896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776553.051898:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776553.051906:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776553.051911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776553.051916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776553.051919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776553.051922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776553.051924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776553.051988:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776553.051990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776553.051992:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776554.011865:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776554.011892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776554.011894:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776554.011898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776554.011900:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776554.011903:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776554.011905:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776554.011906:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776554.011908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776554.075878:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776554.075885:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776554.075887:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776554.075891:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776554.075895:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776554.075899:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776554.075901:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776554.075905:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776554.075910:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776554.075973:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776554.075976:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776554.075979:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776555.035914:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776555.035921:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776555.035923:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776555.035928:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776555.035931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776555.035933:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776555.035935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776555.035937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776555.035939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776555.099898:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776555.099902:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776555.099904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776555.099908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776555.099911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776555.099913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776555.099915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776555.099917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776555.099920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776555.099965:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776555.099969:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776555.099972:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776556.059886:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776556.059891:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776556.059893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776556.059897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776556.059899:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776556.059902:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.059904:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776556.059906:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.059908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776556.123887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776556.123894:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776556.123897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776556.123902:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776556.123908:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776556.123911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776556.123916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776556.123920:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776556.123924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776556.123973:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776556.123977:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776556.123980:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776556.380082:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776556.380087:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776556.380090:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776556.380097:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380110:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776556.380113:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.380116:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776556.380118:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776556.380119:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776556.380125:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099765632:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776556.380131:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776556.380134:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776556.380138:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.380141:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776556.380146:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776556.380150:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776556.380157:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776556.380160:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776556.380165:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776556.380173:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099765632, portal 25 00000100:00000001:3.0:1775776556.380177:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776556.380180:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716032 : -114841420835584 : ffff978d65e7d500) 00000100:00000040:3.0:1775776556.380186:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380199:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776556.380205:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776556.380210:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776556.380216:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099765632, offset 0 00000400:00000200:3.0:1775776556.380227:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776556.380238:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.380247:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776556.380255:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776556.380263:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776556.380270:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776556.380276:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776556.380283:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4212:4212:256:4294967295] 192.168.201.139@tcp LPNI seq info [4212:4212:8:4294967295] 00000400:00000200:3.0:1775776556.380287:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.380296:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776556.380302:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.380307:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776556.380316:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776556.380321:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776556.380329:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776556.380332:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776556.380337:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.380340:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776556.380346:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776556.380352:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776556.380355:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776556.380359:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776556.380366:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776556.380371:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380387:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.380391:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380406:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.380407:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776556.380413:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776556.380420:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776556.380423:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4312945397 00000100:00000010:3.0:1775776556.380428:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344360. 00000100:00000001:3.0:1775776556.380436:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776556.380445:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4312945397 00000100:00000010:3.0:1775776556.380449:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344c60. 00000100:00000001:3.0:1775776556.380453:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776556.380458:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776556.380461:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.380465:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776556.380472:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776556/real 1775776556] req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776556.380486:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776556.380489:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776556.380491:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776556.380494:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776556.380496:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776556.380499:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776556.380504:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776556.380512:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380520:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776556.380523:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.380526:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.380529:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776556.380534:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380551:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776556.380560:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380576:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776556.380581:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.380584:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.380586:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.380590:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776556.380593:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776556.380597:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776556.380606:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776556.380611:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776556.380617:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776556.380630:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099765632:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776556.380636:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.380638:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.380644:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d500 x1862033099765632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380660:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776556.380663:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776556.380666:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776556.380670:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776556.380673:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776556.380676:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776556.380678:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776556.380679:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:3.0:1775776556.380682:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776556.380685:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776556.380686:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d500. 00000100:00000001:3.0:1775776556.380691:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776556.380693:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.380698:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.380707:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776556.380711:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.380717:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776556.380719:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776556.380721:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776556.380727:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380740:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776556.380741:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.380745:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776556.380747:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776556.380750:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776556.380754:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099765760:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776556.380760:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776556.380763:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776556.380765:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.380767:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776556.380771:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776556.380776:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776556.380797:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776556.380799:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776556.380802:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776556.380805:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099765760, portal 10 00000100:00000001:3.0:1775776556.380806:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776556.380807:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288723200 : -114841420828416 : ffff978d65e7f100) 00000100:00000040:3.0:1775776556.380816:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380824:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776556.380842:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776556.380846:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776556.380851:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099765760, offset 0 00000400:00000200:3.0:1775776556.380859:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776556.380865:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.380871:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776556.380876:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776556.380880:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776556.380884:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776556.380886:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776556.380891:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4213:4213:256:4294967295] 192.168.201.139@tcp LPNI seq info [4213:4213:8:4294967295] 00000400:00000200:3.0:1775776556.380898:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.380911:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776556.380925:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.380929:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776556.380935:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776556.380944:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776556.380953:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776556.380956:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776556.380964:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.380966:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776556.380969:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776556.380971:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776556.380974:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776556.380975:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776556.380978:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776556.380981:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380988:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.380990:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.380999:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381000:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776556.381003:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776556.381006:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776556.381010:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776556.381011:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312944133 00000100:00000001:3.0:1775776556.381013:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312945397 00000100:00000001:3.0:1775776556.381015:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312944133 00000100:00000001:3.0:1775776556.381017:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312945397 00000100:00000001:3.0:1775776556.381018:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776556.381020:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381021:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776556.381024:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776556/real 1775776556] req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776556.381031:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776556.381032:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776556.381033:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776556.381035:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776556.381036:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776556.381038:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776556.381041:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776556.381048:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381054:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776556.381056:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381057:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.381058:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776556.381060:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381064:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776556.381066:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381070:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776556.381072:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381072:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.381073:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381074:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776556.381075:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776556.381076:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776556.381078:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776556.381079:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776556.381081:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776556.381085:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099765760:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776556.381088:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776556.381089:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381094:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776556.381094:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.381095:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776556.381096:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776556.381098:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776556.381099:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099765888:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776556.381101:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776556.381102:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776556.381103:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.381104:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776556.381107:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776556.381108:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776556.381110:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776556.381112:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776556.381114:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776556.381115:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099765888, portal 10 00000100:00000001:3.0:1775776556.381116:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776556.381117:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724096 : -114841420827520 : ffff978d65e7f480) 00000100:00000040:3.0:1775776556.381121:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381124:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776556.381126:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776556.381127:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776556.381128:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099765888, offset 0 00000400:00000200:3.0:1775776556.381130:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776556.381133:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.381135:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776556.381136:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776556.381138:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776556.381139:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776556.381141:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776556.381143:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4214:4214:256:4294967295] 192.168.201.139@tcp LPNI seq info [4214:4214:8:4294967295] 00000400:00000200:3.0:1775776556.381145:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.381148:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776556.381151:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.381152:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776556.381155:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776556.381156:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776556.381158:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776556.381160:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776556.381162:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.381164:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776556.381165:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776556.381167:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776556.381169:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776556.381170:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776556.381171:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776556.381173:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381178:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.381179:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381183:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381184:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776556.381185:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776556.381187:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776556.381188:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776556.381189:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312944134 00000100:00000001:3.0:1775776556.381191:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312945397 00000100:00000001:3.0:1775776556.381192:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312944134 00000100:00000001:3.0:1775776556.381193:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312945397 00000100:00000001:3.0:1775776556.381195:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776556.381196:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381197:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776556.381200:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776556/real 1775776556] req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776556.381207:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776556.381208:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776556.381209:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776556.381211:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776556.381213:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776556.381214:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776556.381217:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776556.381225:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381231:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776556.381232:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381233:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.381234:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776556.381238:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381244:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776556.381247:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381251:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776556.381252:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381253:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.381254:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381255:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776556.381256:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776556.381258:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776556.381259:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776556.381260:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776556.381262:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776556.381266:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099765888:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776556.381269:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776556.381271:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381274:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776556.381275:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.381277:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776556.381278:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776556.381279:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776556.381280:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099766016:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776556.381282:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776556.381283:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776556.381284:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.381285:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776556.381287:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776556.381288:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776556.381291:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776556.381292:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776556.381294:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776556.381296:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099766016, portal 4 00000100:00000001:3.0:1775776556.381297:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776556.381298:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716928 : -114841420834688 : ffff978d65e7d880) 00000100:00000040:3.0:1775776556.381302:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381305:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776556.381306:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776556.381308:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776556.381309:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099766016, offset 0 00000400:00000200:3.0:1775776556.381311:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776556.381314:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.381317:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776556.381319:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776556.381321:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776556.381323:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776556.381326:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776556.381328:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4215:4215:256:4294967295] 192.168.201.139@tcp LPNI seq info [4215:4215:8:4294967295] 00000400:00000200:3.0:1775776556.381332:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.381337:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776556.381341:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.381344:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776556.381347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776556.381350:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776556.381353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776556.381356:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776556.381360:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.381361:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776556.381364:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776556.381367:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776556.381369:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776556.381371:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776556.381373:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776556.381375:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381380:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.381382:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381385:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381386:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776556.381388:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776556.381390:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776556.381392:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776556.381393:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312944143 00000100:00000001:3.0:1775776556.381395:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312945397 00000100:00000001:3.0:1775776556.381396:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312944143 00000100:00000001:3.0:1775776556.381398:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312945397 00000100:00000001:3.0:1775776556.381399:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776556.381400:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381401:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776556.381403:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776556/real 1775776556] req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776556.381408:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776556.381409:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776556.381409:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776556.381410:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776556.381411:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776556.381412:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776556.381414:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776556.381419:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381422:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776556.381423:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381423:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.381424:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776556.381426:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381429:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776556.381431:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381434:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776556.381436:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381436:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.381437:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381438:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776556.381439:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776556.381440:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776556.381442:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776556.381443:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776556.381445:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776556.381449:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099766016:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776556.381451:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381452:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.381454:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f100 x1862033099765760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381457:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776556.381458:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776556.381459:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776556.381460:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776556.381461:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776556.381462:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776556.381463:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776556.381464:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:3.0:1775776556.381466:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776556.381467:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776556.381467:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f100. 00000100:00000001:3.0:1775776556.381469:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776556.381470:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381471:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.381472:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f480 x1862033099765888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381476:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776556.381477:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776556.381477:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776556.381479:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776556.381479:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776556.381480:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776556.381481:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776556.381482:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:3.0:1775776556.381483:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776556.381484:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776556.381484:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f480. 00000100:00000001:3.0:1775776556.381486:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776556.381486:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381487:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.381488:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d880 x1862033099766016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381492:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776556.381492:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776556.381493:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776556.381494:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776556.381494:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776556.381495:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776556.381496:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776556.381496:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:3.0:1775776556.381497:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776556.381498:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776556.381498:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d880. 00000100:00000001:3.0:1775776556.381499:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776556.381500:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381501:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381503:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776556.381504:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381505:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776556.381506:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776556.381507:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776556.381510:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381515:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776556.381517:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.381518:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776556.381519:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776556.381521:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776556.381523:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099766144:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776556.381527:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776556.381528:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776556.381529:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776556.381531:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776556.381534:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776556.381535:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776556.381538:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776556.381540:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776556.381543:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776556.381546:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099766144, portal 4 00000100:00000001:3.0:1775776556.381547:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776556.381548:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288717824 : -114841420833792 : ffff978d65e7dc00) 00000100:00000040:3.0:1775776556.381553:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381558:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776556.381560:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776556.381562:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776556.381564:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099766144, offset 0 00000400:00000200:3.0:1775776556.381568:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776556.381572:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.381575:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776556.381577:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776556.381579:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776556.381581:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776556.381584:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776556.381586:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4216:4216:256:4294967295] 192.168.201.139@tcp LPNI seq info [4216:4216:8:4294967295] 00000400:00000200:3.0:1775776556.381590:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776556.381595:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776556.381598:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.381600:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776556.381603:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776556.381605:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776556.381610:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776556.381612:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776556.381615:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776556.381617:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776556.381619:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776556.381623:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776556.381625:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776556.381627:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776556.381629:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776556.381632:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381639:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.381641:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776581 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381645:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381646:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776556.381648:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776556.381650:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776556.381652:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776556.381653:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4312944134 00000100:00000001:3.0:1775776556.381655:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4312945397 00000100:00000001:3.0:1775776556.381656:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4312944134 00000100:00000001:3.0:1775776556.381657:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4312945397 00000100:00000001:3.0:1775776556.381658:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776556.381659:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381660:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776556.381662:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776556/real 1775776556] req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776556.381666:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776556.381667:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776556.381668:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776556.381669:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776556.381670:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776556.381672:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776556.381673:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776556.381678:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381681:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776556.381681:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381682:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.381683:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776556.381685:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381688:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776556.381690:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381694:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776556.381695:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381696:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776556.381697:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381698:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776556.381699:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776556.381700:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776556.381702:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776556.381703:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776556.381705:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776556.381709:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099766144:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776556.381711:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381712:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776556.381714:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7dc00 x1862033099766144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776581 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776556.381717:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776556.381718:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776556.381719:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776556.381720:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776556.381721:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776556.381721:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776556.381722:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776556.381723:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505c00. 02000000:00000001:3.0:1775776556.381724:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776556.381725:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776556.381726:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7dc00. 00000100:00000001:3.0:1775776556.381727:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776556.381728:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381729:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776556.381731:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776556.381732:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776556.381733:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776556.381734:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776557.083886:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776557.083891:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776557.083892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776557.083897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776557.083900:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776557.083903:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776557.083905:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776557.083907:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776557.083908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776557.148871:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776557.148875:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776557.148878:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776557.149398:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776557.149401:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776557.149403:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776557.149406:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776557.149410:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776557.149413:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776557.149416:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776557.149418:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776557.149421:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776557.659907:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312945397, at 4312945408 00000100:00000040:1.0:1775776557.659907:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312945397, at 4312945408 00000100:00000010:0.0:1775776557.659920:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344360. 00000100:00000010:1.0:1775776557.659923:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c60. 00000100:00000001:3.0:1775776558.107900:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776558.107905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776558.107906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776558.107910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776558.107913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776558.107916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776558.107917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776558.107919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776558.107921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776558.171899:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776558.171905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776558.171906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776558.171912:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776558.171915:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776558.171918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776558.171923:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776558.171926:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776558.171931:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776558.171990:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776558.171994:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776558.171996:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776559.131886:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776559.131892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776559.131893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776559.131898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776559.131901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776559.131904:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776559.131907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776559.131909:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776559.131911:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776559.195935:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776559.195938:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776559.195940:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776559.195943:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776559.195946:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776559.195949:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776559.195951:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776559.195953:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776559.195955:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776559.195977:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776559.195978:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776559.195980:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776560.155898:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776560.155905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776560.155907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776560.155912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776560.155916:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776560.155919:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776560.155922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776560.155924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776560.155927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776560.219877:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776560.219882:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776560.219885:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776560.219892:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776560.219896:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776560.219900:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776560.219903:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776560.219905:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776560.219909:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776560.219986:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776560.219988:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776560.219991:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776561.179844:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776561.179850:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776561.179851:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776561.179855:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776561.179857:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776561.179860:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776561.179862:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776561.179863:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776561.179865:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776561.243861:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776561.243867:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776561.243869:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776561.243874:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776561.243878:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776561.243881:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776561.243884:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776561.243886:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776561.243891:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776561.243965:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776561.243970:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776561.243973:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776561.499916:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776561.499925:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776561.499927:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776561.499931:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776561.499935:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776561.499942:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18276 00000400:00000200:3.0:1775776561.499953:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776561.499959:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776561.499962:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776561.499965:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.499967:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.499970:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.499972:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776561.499976:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776561.499977:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776561.499980:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776561.499988:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776561.499991:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776561.499993:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776561.499996:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.499997:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.499999:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500001:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776561.500004:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776561.500007:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500009:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776561.500012:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776561.500016:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776561.500019:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500021:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776561.500023:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776561.500033:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776561.500036:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776561.500040:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776561.500042:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500045:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776561.500047:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776561.500051:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776561.500065:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776561.500071:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776561.500080:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099766272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776561.500090:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776561.500094:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776561.500099:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099766272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776561.500120:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776561.500123:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776561.500126:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776561.500128:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776561.500130:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776561.500132:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776561.500134:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18276 00000400:00000200:3.0:1775776561.500144:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776561.500147:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776561.500149:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776561.500152:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500154:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.500156:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500158:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776561.500160:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776561.500161:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776561.500163:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776561.500165:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776561.500166:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776561.500168:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776561.500171:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500173:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.500175:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500177:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776561.500179:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776561.500182:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500184:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776561.500186:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776561.500191:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776561.500193:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500196:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776561.500200:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776561.500211:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500215:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776561.500219:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776561.500224:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776561.500229:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776561.500231:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500235:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776561.500237:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776561.500240:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776561.500245:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775776561.500249:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776561.500254:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099766400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776561.500266:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776561.500269:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776561.500275:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099766400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776561.500290:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776561.500294:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776561.500300:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776561.500302:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776561.500304:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776561.500308:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776561.500313:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18276 00000400:00000200:3.0:1775776561.500320:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776561.500326:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776561.500329:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776561.500333:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500345:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.500350:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500353:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776561.500356:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776561.500358:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776561.500361:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776561.500365:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776561.500370:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776561.500374:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776561.500380:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500384:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.500387:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500392:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776561.500395:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776561.500408:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500415:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776561.500419:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776561.500427:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776561.500431:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500435:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776561.500438:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776561.500443:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500447:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776561.500453:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776561.500458:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776561.500463:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776561.500465:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500467:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776561.500469:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776561.500471:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776561.500477:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776561.500482:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776561.500490:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099766528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776561.500501:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776561.500505:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776561.500511:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099766528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776561.500523:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776561.500527:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776561.500528:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00080000:3.0:1775776561.500532:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776561.500539:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776561.500542:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776561.500544:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776561.500547:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776561.500551:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18276 00000400:00000200:3.0:1775776561.500560:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776561.500565:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776561.500568:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776561.500573:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500576:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.500581:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500585:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776561.500588:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776561.500590:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776561.500591:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776561.500593:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776561.500595:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776561.500599:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776561.500605:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500610:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.500613:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500616:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776561.500632:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776561.500637:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500641:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776561.500645:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776561.500649:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776561.500652:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500654:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776561.500657:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776561.500661:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500663:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776561.500670:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776561.500672:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776561.500680:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776561.500682:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500687:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776561.500688:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776561.500690:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776561.500696:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776561.500702:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776561.500709:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099766656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776561.500717:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776561.500719:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776561.500726:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099766656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776561.500752:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776561.500757:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776561.500763:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776561.500765:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776561.500768:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776561.500773:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776561.500778:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18276 00000400:00000200:3.0:1775776561.500818:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776561.500824:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776561.500828:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776561.500833:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500836:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.500840:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500844:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776561.500847:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776561.500850:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776561.500852:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776561.500854:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776561.500858:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776561.500860:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776561.500864:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500865:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776561.500868:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500870:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776561.500872:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776561.500876:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776561.500879:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776561.500883:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776561.500886:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776561.500888:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500892:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776561.500894:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776561.500898:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500902:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776561.500906:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776561.500908:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776561.500911:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776561.500913:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776561.500915:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776561.500916:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776561.500918:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776561.500924:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776561.500928:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776561.500933:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099766784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776561.500947:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776561.500951:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776561.500959:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099766784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776561.500974:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776561.500979:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18286) 00000800:00000200:2.1:1775776561.501227:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776561.501229:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776561.501488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776561.501490:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776561.501769:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776561.501771:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776561.502040:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776561.502042:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1775776562.203887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776562.203894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776562.203895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776562.203899:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776562.203902:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776562.203905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776562.203906:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776562.203908:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776562.203910:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776562.267887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776562.267892:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776562.267894:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776562.267898:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776562.267901:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776562.267904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776562.267906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776562.267908:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776562.267918:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776562.267997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776562.268001:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776562.268004:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776562.779854:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312950517, at 4312950528 00000100:00000040:1.0:1775776562.779854:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312950517, at 4312950528 00000100:00000010:0.0:1775776562.779869:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000010:1.0:1775776562.779871:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000001:3.0:1775776563.227885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776563.227890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776563.227891:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776563.227895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776563.227898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776563.227901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776563.227903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776563.227904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776563.227906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776563.291869:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776563.291874:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776563.291878:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776563.291883:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776563.291890:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776563.291895:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776563.291900:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776563.291903:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776563.291906:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776563.291955:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776563.291959:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776563.291961:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776564.251897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776564.251903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776564.251904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776564.251909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776564.251911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776564.251914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776564.251916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776564.251918:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776564.251919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776564.315910:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776564.315917:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776564.315921:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776564.315927:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776564.315943:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776564.315945:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776564.315947:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776564.315949:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776564.315951:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776564.316026:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776564.316030:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776564.316033:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776565.275874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776565.275879:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776565.275881:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776565.275884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776565.275887:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776565.275889:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776565.275891:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776565.275893:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776565.275895:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776565.339852:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776565.339859:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776565.339860:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776565.339864:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776565.339868:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776565.339872:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776565.339876:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776565.339879:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776565.339884:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776565.339953:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776565.339957:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776565.339960:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776566.299884:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776566.299890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776566.299891:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776566.299895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776566.299897:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776566.299900:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776566.299902:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776566.299903:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776566.299905:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776566.363885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776566.363891:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776566.363892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776566.363896:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776566.363900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776566.363902:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776566.363905:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776566.363908:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776566.363912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776566.363988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776566.363991:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776566.363994:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776566.619851:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776566.619859:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776566.619861:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776566.619865:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776566.619868:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776566.619873:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18281 00000400:00000200:3.0:1775776566.619880:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776566.619888:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776566.619891:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776566.619893:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.619895:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.619896:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.619898:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776566.619901:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776566.619902:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776566.619904:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776566.619906:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776566.619907:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776566.619908:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776566.619911:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.619912:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.619913:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.619914:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776566.619915:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776566.619917:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776566.619919:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776566.619921:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776566.619922:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776566.619924:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.619926:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776566.619927:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776566.619933:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776566.619936:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776566.619939:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776566.619941:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.619944:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776566.619946:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776566.619950:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776566.619958:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776566.619964:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776566.619973:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099766912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776566.619983:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776566.619988:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776566.619993:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099766912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776566.620011:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776566.620014:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776566.620019:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776566.620020:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776566.620021:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776566.620023:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776566.620024:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18281 00000400:00000200:3.0:1775776566.620028:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776566.620031:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776566.620033:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776566.620035:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620037:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.620038:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620041:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776566.620043:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776566.620045:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776566.620046:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776566.620049:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776566.620051:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776566.620053:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776566.620056:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620060:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.620063:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620066:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776566.620069:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776566.620072:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620075:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776566.620076:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776566.620081:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776566.620085:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620088:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776566.620092:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776566.620095:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620097:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776566.620103:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776566.620107:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776566.620113:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776566.620114:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620119:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776566.620122:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776566.620125:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776566.620131:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776566.620135:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776566.620141:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099767040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776566.620152:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776566.620156:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776566.620163:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099767040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776566.620183:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776566.620186:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776566.620188:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776566.620190:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776566.620191:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776566.620193:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776566.620196:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18281 00000400:00000200:3.0:1775776566.620225:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776566.620231:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776566.620234:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776566.620241:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620244:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.620249:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620250:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776566.620253:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776566.620254:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776566.620254:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776566.620256:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776566.620259:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776566.620263:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776566.620270:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776566.620272:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776566.620274:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775776566.620275:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000100:00000040:3.0:1775776566.620278:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620282:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776566.620286:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776566.620291:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620293:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776566.620295:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776566.620298:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776566.620301:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620305:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776566.620309:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776566.620314:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620317:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776566.620323:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776566.620329:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776566.620335:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776566.620338:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620341:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776566.620344:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776566.620346:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776566.620351:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776566.620354:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776566.620362:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099767168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776566.620370:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776566.620372:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776566.620375:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099767168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776566.620381:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776566.620385:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776566.620390:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776566.620392:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776566.620394:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776566.620399:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776566.620410:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18281 00000400:00000200:3.0:1775776566.620418:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776566.620425:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776566.620430:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776566.620436:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620441:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.620445:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620448:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776566.620452:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776566.620454:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776566.620456:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776566.620458:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776566.620460:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776566.620462:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776566.620468:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620471:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.620475:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620479:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776566.620481:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776566.620485:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620487:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776566.620489:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776566.620492:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776566.620495:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620498:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776566.620501:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776566.620505:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620508:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776566.620512:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776566.620516:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776566.620520:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776566.620522:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620525:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776566.620528:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776566.620530:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776566.620534:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776566.620538:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776566.620545:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099767296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776566.620568:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776566.620571:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776566.620578:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099767296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776566.620597:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776566.620601:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776566.620605:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776566.620607:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776566.620609:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776566.620612:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776566.620615:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18281 00000400:00000200:3.0:1775776566.620621:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776566.620626:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776566.620628:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776566.620633:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620637:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.620639:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620640:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776566.620641:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776566.620644:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776566.620646:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776566.620660:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776566.620663:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776566.620666:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776566.620670:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620672:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776566.620676:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620680:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776566.620684:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776566.620689:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776566.620692:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776566.620694:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776566.620697:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776566.620699:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620700:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776566.620701:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776566.620703:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620708:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776566.620711:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776566.620714:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776566.620719:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776566.620721:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776566.620723:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776566.620724:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776566.620727:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776566.620733:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776566.620737:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776566.620743:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099767424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776566.620756:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776566.620757:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776566.620759:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099767424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776566.620768:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776566.620774:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18292) 00000800:00000200:2.1:1775776566.620837:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776566.620839:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776566.621000:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776566.621011:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776566.621163:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776566.621164:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776566.621367:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776566.621369:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000100:00000001:3.0:1775776567.323913:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776567.323919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776567.323920:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776567.323932:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776567.323936:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776567.323940:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776567.323942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776567.323944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776567.323947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776567.387913:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776567.387919:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776567.387923:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776567.387928:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776567.387934:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776567.387938:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776567.387940:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776567.387943:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776567.387946:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776567.387986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776567.387989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776567.387992:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776567.899894:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312955637, at 4312955648 00000100:00000040:1.0:1775776567.899894:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312955637, at 4312955648 00000100:00000010:0.0:1775776567.899909:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000010:1.0:1775776567.899910:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775776568.347895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776568.347901:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776568.347903:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776568.347907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776568.347910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776568.347913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776568.347915:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776568.347917:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776568.347919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776568.411918:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776568.411922:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776568.411925:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776568.411929:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776568.411932:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776568.411936:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776568.411938:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776568.411940:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776568.411942:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776568.411991:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775776568.411995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775776568.411998:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776569.371881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776569.371886:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776569.371888:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776569.371891:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776569.371894:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776569.371898:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776569.371899:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776569.371901:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776569.371903:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776569.435875:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776569.435879:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776569.435881:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776569.435885:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776569.435887:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776569.435890:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776569.435892:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776569.435893:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776569.435895:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776569.435932:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776569.435936:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776569.435939:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776570.395854:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776570.395861:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776570.395863:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776570.395868:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776570.395871:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776570.395875:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776570.395878:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776570.395880:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776570.395883:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776570.459942:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776570.459947:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776570.459951:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776570.459957:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776570.459962:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776570.459967:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776570.459971:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776570.459975:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776570.459980:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776570.460015:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776570.460018:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776570.460022:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776571.419870:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776571.419876:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776571.419878:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776571.419882:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776571.419884:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776571.419887:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776571.419889:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776571.419890:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776571.419892:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776571.483871:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776571.483875:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776571.483877:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776571.483881:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776571.483885:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776571.483889:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776571.483891:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776571.483893:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776571.483895:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776571.483939:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776571.483942:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776571.483945:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776571.739877:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776571.739884:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776571.739886:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776571.739889:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776571.739893:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776571.739897:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18287 00000400:00000200:3.0:1775776571.739906:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776571.739910:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776571.739913:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776571.739915:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.739917:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.739919:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.739920:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776571.739923:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776571.739924:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776571.739927:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776571.739929:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776571.739931:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776571.739933:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776571.739936:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.739937:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.739939:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.739941:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776571.739944:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776571.739947:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776571.739950:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776571.739953:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776571.739956:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776571.739958:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.739961:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776571.739963:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776571.739973:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775776571.739977:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776571.739981:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776571.739982:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.739984:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776571.739986:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776571.739989:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776571.739994:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776571.739999:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776571.740006:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099767552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776571.740013:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776571.740016:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776571.740019:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099767552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776571.740037:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776571.740039:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776571.740041:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776571.740042:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776571.740043:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776571.740045:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776571.740046:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18287 00000400:00000200:3.0:1775776571.740049:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776571.740051:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776571.740053:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776571.740055:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740057:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.740059:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740067:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776571.740069:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776571.740071:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776571.740073:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776571.740075:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776571.740078:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776571.740080:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776571.740085:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740089:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.740092:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740096:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776571.740099:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776571.740102:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740105:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776571.740106:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776571.740111:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776571.740115:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740118:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776571.740123:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776571.740125:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740128:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776571.740134:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776571.740136:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776571.740138:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776571.740139:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740143:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776571.740143:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776571.740145:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776571.740148:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776571.740150:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776571.740157:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099767680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776571.740166:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776571.740170:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776571.740174:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099767680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776571.740186:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776571.740191:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776571.740196:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776571.740199:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776571.740202:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776571.740205:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776571.740208:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18287 00000400:00000200:3.0:1775776571.740214:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776571.740220:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776571.740224:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776571.740229:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740233:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.740235:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740239:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776571.740241:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776571.740244:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776571.740247:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776571.740250:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776571.740252:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776571.740254:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776571.740260:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740262:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.740267:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740271:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776571.740273:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776571.740278:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740282:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776571.740284:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776571.740286:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776571.740291:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740294:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776571.740298:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776571.740304:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740307:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776571.740323:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776571.740327:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776571.740332:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776571.740334:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740338:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776571.740340:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776571.740343:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776571.740350:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000800:00000200:2.1:1775776571.740350:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776571.740355:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776571.740356:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000100:00100000:3.0:1775776571.740362:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099767808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776571.740372:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776571.740375:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776571.740378:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099767808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776571.740387:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776571.740391:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776571.740396:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776571.740397:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776571.740398:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776571.740402:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776571.740407:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18287 00000400:00000200:3.0:1775776571.740411:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776571.740416:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776571.740420:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776571.740425:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740428:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.740430:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740433:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776571.740435:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776571.740437:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776571.740439:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776571.740441:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776571.740444:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776571.740448:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776571.740451:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740454:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.740456:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740458:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776571.740463:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776571.740466:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740468:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776571.740470:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776571.740474:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776571.740475:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740477:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776571.740480:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776571.740482:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740484:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776571.740488:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776571.740490:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776571.740492:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776571.740494:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740496:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776571.740497:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776571.740501:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776571.740504:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776571.740508:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776571.740516:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099767936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776571.740529:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776571.740533:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776571.740539:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099767936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776571.740548:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776571.740552:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776571.740556:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776571.740575:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776571.740577:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776571.740579:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776571.740583:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18287 00000400:00000200:3.0:1775776571.740593:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776571.740599:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776571.740610:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776571.740617:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740622:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.740627:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740631:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776571.740636:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776571.740638:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776571.740641:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776571.740643:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776571.740647:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776571.740650:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776571.740657:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740660:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776571.740663:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740666:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776571.740668:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776571.740673:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776571.740676:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776571.740678:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776571.740682:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776571.740686:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740687:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776571.740689:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776571.740695:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740696:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776571.740701:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776571.740704:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776571.740708:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776571.740710:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776571.740712:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776571.740714:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776571.740718:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776571.740721:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776571.740726:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776571.740734:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099768064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776571.740746:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776571.740749:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776571.740752:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099768064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776571.740764:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776571.740768:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18297) 00000800:00000200:2.1:1775776571.740867:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776571.740869:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776571.741035:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776571.741037:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776571.741261:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776571.741263:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776571.741518:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776571.741521:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1775776572.443893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776572.443898:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776572.443899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776572.443903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776572.443905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776572.443908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776572.443910:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776572.443911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776572.443913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776572.507889:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776572.507896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776572.507899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776572.507906:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776572.507911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776572.507915:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776572.507919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776572.507923:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776572.507926:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776572.507988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776572.507992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776572.507995:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776573.019895:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312960757, at 4312960768 00000100:00000010:0.0:1775776573.019912:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000040:1.0:1775776573.020075:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312960757, at 4312960768 00000100:00000010:1.0:1775776573.020082:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000001:3.0:1775776573.467896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776573.467904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776573.467906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776573.467915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776573.467919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776573.467923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776573.467925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776573.467927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776573.467930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776573.531872:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776573.531877:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776573.531879:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776573.531885:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776573.531889:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776573.531892:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776573.531896:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776573.531898:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776573.531901:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776573.531946:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776573.531949:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776573.531952:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776574.491881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776574.491887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776574.491888:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776574.491892:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776574.491894:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776574.491897:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776574.491899:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776574.491901:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776574.491902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776574.555893:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776574.555900:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776574.555901:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776574.555907:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776574.555910:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776574.555914:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776574.555916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776574.555918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776574.555921:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776574.556033:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776574.556037:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776574.556040:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776575.515883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776575.515888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776575.515890:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776575.515894:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776575.515896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776575.515899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776575.515902:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776575.515904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776575.515905:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776575.579892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776575.579901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776575.579904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776575.579910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776575.579916:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776575.579920:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776575.579924:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776575.579927:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776575.579930:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776575.579991:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776575.579995:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776575.579998:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776576.539885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776576.539892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776576.539894:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776576.539899:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776576.539902:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776576.539906:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776576.539909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776576.539911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776576.539914:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776576.603870:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776576.603878:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776576.603883:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776576.603890:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776576.603896:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776576.603900:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776576.603906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776576.603910:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776576.603915:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776576.603965:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776576.603967:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776576.603970:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776576.859939:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776576.859953:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776576.859956:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776576.859961:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776576.859964:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776576.859971:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18292 00000400:00000200:1.0:1775776576.859981:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776576.859987:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776576.859990:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776576.859993:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.859997:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.859999:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860001:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776576.860005:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776576.860006:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776576.860008:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776576.860010:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776576.860012:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776576.860013:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776576.860015:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860016:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860017:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860018:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776576.860020:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776576.860022:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860024:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776576.860027:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776576.860029:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776576.860031:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860032:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776576.860034:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776576.860042:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776576.860044:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776576.860047:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776576.860048:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860050:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776576.860052:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776576.860055:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776576.860061:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776576.860065:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776576.860072:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099768192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776576.860079:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776576.860083:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776576.860085:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099768192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776576.860103:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776576.860105:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776576.860107:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776576.860108:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776576.860109:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776576.860111:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776576.860113:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18292 00000400:00000200:1.0:1775776576.860122:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776576.860123:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776576.860125:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776576.860127:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860128:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860130:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860131:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776576.860133:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776576.860134:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776576.860135:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776576.860137:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776576.860138:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776576.860139:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776576.860140:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860142:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860143:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860144:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776576.860145:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776576.860147:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860149:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776576.860151:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776576.860155:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776576.860157:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860160:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776576.860163:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776576.860165:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860168:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776576.860172:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776576.860175:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776576.860179:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776576.860181:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860184:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776576.860186:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776576.860187:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776576.860192:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776576.860195:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776576.860212:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099768320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776576.860225:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776576.860227:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776576.860232:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099768320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776576.860248:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776576.860252:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776576.860255:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776576.860257:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776576.860258:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776576.860262:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776576.860265:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18292 00000400:00000200:1.0:1775776576.860270:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776576.860281:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776576.860285:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776576.860290:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860293:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860297:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860301:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776576.860305:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776576.860307:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776576.860310:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776576.860312:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776576.860315:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776576.860319:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776576.860324:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860328:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860330:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860335:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776576.860338:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776576.860347:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860349:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776576.860352:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776576.860355:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776576.860359:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860362:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776576.860365:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776576.860368:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860371:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776576.860375:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776576.860378:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776576.860382:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776576.860383:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860385:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776576.860386:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776576.860388:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776576.860391:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776576.860397:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776576.860405:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099768448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776576.860424:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776576.860426:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776576.860432:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099768448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776576.860448:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776576.860453:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776576.860457:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776576.860458:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776576.860459:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776576.860461:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776576.860465:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18292 00000400:00000200:1.0:1775776576.860479:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776576.860484:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776576.860486:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776576.860491:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860492:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860496:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776576.860496:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776576.860497:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776576.860500:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776576.860502:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776576.860503:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776576.860504:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776576.860506:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776576.860507:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776576.860509:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860510:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860511:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860512:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776576.860513:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776576.860514:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860515:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776576.860516:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776576.860518:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776576.860519:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860521:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776576.860527:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776576.860532:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860535:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776576.860541:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776576.860545:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776576.860548:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776576.860550:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860553:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776576.860555:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776576.860557:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776576.860562:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776576.860566:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776576.860573:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099768576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776576.860587:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776576.860590:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776576.860595:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099768576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776576.860610:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776576.860615:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776576.860619:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776576.860621:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776576.860623:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776576.860626:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776576.860631:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18292 00000400:00000200:1.0:1775776576.860638:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776576.860647:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776576.860650:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776576.860655:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860658:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860662:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860664:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776576.860667:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776576.860669:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776576.860671:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776576.860673:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776576.860675:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776576.860678:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776576.860679:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860683:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776576.860686:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860689:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776576.860691:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776576.860694:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776576.860696:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776576.860698:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776576.860700:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776576.860703:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860708:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776576.860710:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776576.860715:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860716:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776576.860721:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776576.860723:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776576.860725:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776576.860726:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776576.860728:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776576.860729:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776576.860731:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776576.860736:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776576.860740:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776576.860746:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099768704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776576.860753:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776576.860761:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776576.860768:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099768704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776576.860809:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776576.860815:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18302) 00000800:00000200:2.1:1775776576.860860:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776576.860998:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776576.861146:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776576.861276:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776577.563898:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776577.563905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776577.563906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776577.563911:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776577.563914:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776577.563918:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776577.563921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776577.563923:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776577.563926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776577.627841:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776577.627847:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776577.627851:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776577.627863:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776577.627868:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776577.627871:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776577.627874:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776577.627876:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776577.627879:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776577.627914:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776577.627916:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776577.627918:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776578.139891:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312965877, at 4312965888 00000100:00000010:1.0:1775776578.139905:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000040:0.0:1775776578.140513:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312965877, at 4312965888 00000100:00000010:0.0:1775776578.140520:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:3.0:1775776578.587850:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776578.587857:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776578.587859:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776578.587863:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776578.587865:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776578.587868:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776578.587871:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776578.587873:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776578.587876:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776578.651965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776578.651969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776578.651973:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776578.652161:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776578.652165:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776578.652167:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776578.652171:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776578.652174:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776578.652178:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776578.652180:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776578.652183:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776578.652185:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.611873:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776579.611880:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776579.611882:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776579.611888:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776579.611892:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776579.611895:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776579.611897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776579.611900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776579.611903:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776579.676015:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776579.676018:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776579.676020:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776579.677403:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776579.677407:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776579.677408:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776579.677412:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776579.677416:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776579.677419:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776579.677431:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776579.677433:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776579.677436:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1775776579.932050:0:301876:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:0.0:1775776579.932090:0:301876:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775776579.932100:0:301876:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776579.932104:0:301876:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:0.0:1775776579.932108:0:301876:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776579.932112:0:301876:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775776579.932117:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776579.932121:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7600. 00000400:00000200:0.0:1775776579.932126:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776579.932130:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776579.932137:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d110 00000400:00000010:0.0:1775776579.932139:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d110. 00000100:00000001:0.0:1775776579.932145:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776579.932152:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65d6d180 x1862033099768192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932181:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776579.932186:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d180 x1862033099768192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932202:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776579.932205:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776579.932214:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776579.932216:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7100. 00000400:00000200:0.0:1775776579.932219:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776579.932222:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776579.932225:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d990 00000400:00000010:0.0:1775776579.932227:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d990. 00000100:00000001:0.0:1775776579.932229:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776579.932233:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65d6ce00 x1862033099768320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932240:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776579.932242:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ce00 x1862033099768320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932253:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776579.932255:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776579.932259:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776579.932261:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7500. 00000400:00000200:0.0:1775776579.932263:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776579.932266:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776579.932268:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d220 00000400:00000010:0.0:1775776579.932270:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d220. 00000100:00000001:0.0:1775776579.932272:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776579.932276:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65d6ed80 x1862033099768448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932283:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776579.932285:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ed80 x1862033099768448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932297:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776579.932298:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776579.932303:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776579.932305:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:0.0:1775776579.932307:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776579.932310:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776579.932313:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d440 00000400:00000010:0.0:1775776579.932314:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d440. 00000100:00000001:0.0:1775776579.932316:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776579.932320:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65d6f480 x1862033099768576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932332:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776579.932335:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f480 x1862033099768576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932341:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776579.932343:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776579.932348:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776579.932351:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:0.0:1775776579.932353:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776579.932356:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776579.932358:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:0.0:1775776579.932360:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:0.0:1775776579.932362:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776579.932365:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65d6c000 x1862033099768704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932373:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776579.932375:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c000 x1862033099768704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776601 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776579.932381:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776579.932383:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776580.635888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776580.635893:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776580.635895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776580.635899:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776580.635901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776580.635911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776580.635913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776580.635916:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776580.635918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776580.699860:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776580.699869:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776580.699872:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776580.699878:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776580.699882:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776580.699886:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776580.699891:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776580.699894:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776580.699898:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776580.699951:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776580.699954:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776580.699956:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776581.659930:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776581.659937:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776581.659949:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776581.659953:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776581.659956:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776581.659959:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776581.659961:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776581.659963:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776581.659965:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776581.723992:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776581.723996:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776581.724000:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776581.724120:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776581.724125:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776581.724127:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776581.724131:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776581.724151:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776581.724155:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776581.724157:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776581.724160:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776581.724162:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776581.979880:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776581.979894:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776581.979897:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776581.979900:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776581.979904:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776581.979909:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18297 00000400:00000200:1.0:1775776581.979917:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776581.979922:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776581.979924:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776581.979926:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.979928:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.979929:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.979931:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776581.979933:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776581.979935:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776581.979937:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776581.979939:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776581.979940:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776581.979942:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776581.979944:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.979944:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.979945:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.979947:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776581.979948:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776581.979950:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776581.979951:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776581.979953:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776581.979955:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776581.979957:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.979958:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776581.979960:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776581.979964:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776581.979966:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776581.979969:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776581.979970:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.979972:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776581.979974:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776581.979976:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776581.979981:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776581.979986:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776581.979994:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099768832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776581.980004:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776581.980008:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776581.980013:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099768832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776581.980042:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776581.980045:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776581.980049:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776581.980050:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776581.980051:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776581.980054:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776581.980056:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18297 00000400:00000200:1.0:1775776581.980060:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776581.980063:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776581.980065:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776581.980068:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980070:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.980072:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980074:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776581.980076:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776581.980077:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776581.980078:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776581.980080:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776581.980082:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776581.980085:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776581.980087:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980089:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.980091:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980093:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776581.980095:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776581.980098:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980100:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776581.980101:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776581.980119:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776581.980122:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980124:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776581.980128:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776581.980131:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980133:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776581.980136:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776581.980139:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776581.980141:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776581.980142:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980144:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776581.980145:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776581.980147:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776581.980150:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776581.980154:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776581.980158:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099768960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776581.980165:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776581.980167:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776581.980171:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099768960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776581.980178:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776581.980180:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776581.980184:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776581.980185:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776581.980186:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776581.980189:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776581.980191:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18297 00000400:00000200:1.0:1775776581.980195:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776581.980198:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776581.980200:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776581.980203:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980205:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.980208:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980209:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776581.980212:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776581.980214:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776581.980215:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776581.980217:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776581.980225:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776581.980230:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776581.980234:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980236:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.980238:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980239:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776581.980243:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776581.980248:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980253:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776581.980257:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776581.980261:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776581.980266:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980270:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776581.980273:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776581.980277:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980281:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776581.980285:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776581.980290:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776581.980294:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776581.980296:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980301:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776581.980302:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776581.980306:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776581.980310:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776581.980316:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776581.980324:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099769088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776581.980338:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776581.980341:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776581.980347:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099769088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776581.980359:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776581.980362:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776581.980368:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776581.980370:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776581.980373:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776581.980376:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776581.980412:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18297 00000400:00000200:1.0:1775776581.980418:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776581.980423:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776581.980427:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776581.980431:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980435:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.980439:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980441:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776581.980443:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776581.980444:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776581.980446:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776581.980450:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776581.980453:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776581.980454:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776581.980457:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980461:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.980466:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980468:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776581.980473:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776581.980478:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980481:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776581.980484:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776581.980488:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776581.980493:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980497:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776581.980501:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776581.980506:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980509:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776581.980514:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776581.980517:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776581.980523:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776581.980530:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980534:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776581.980537:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776581.980539:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776581.980544:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776581.980549:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776581.980553:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099769216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776581.980564:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776581.980566:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776581.980571:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099769216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776581.980578:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776581.980584:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776581.980584:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00080000:1.0:1775776581.980589:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776581.980596:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776581.980598:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776581.980600:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776581.980604:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776581.980610:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18297 00000400:00000200:1.0:1775776581.980617:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776581.980625:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776581.980627:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776581.980631:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980635:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.980639:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980641:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776581.980645:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776581.980647:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776581.980650:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776581.980653:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776581.980657:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776581.980662:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776581.980669:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980672:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776581.980677:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980681:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776581.980686:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776581.980692:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776581.980696:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776581.980701:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776581.980705:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776581.980707:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980709:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776581.980711:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776581.980715:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980719:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776581.980726:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776581.980730:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776581.980733:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776581.980735:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776581.980739:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776581.980740:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776581.980742:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776581.980746:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776581.980749:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776581.980754:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099769344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776581.980761:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776581.980763:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776581.980766:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099769344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776581.980777:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776581.980810:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18307) 00000800:00000200:2.1:1775776581.981082:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776581.981085:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776581.981430:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776581.981458:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776581.981716:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776581.981719:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776581.981994:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776581.981997:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1775776582.683858:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776582.683863:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776582.683864:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776582.683869:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776582.683871:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776582.683874:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776582.683876:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776582.683878:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776582.683879:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776582.747843:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776582.747855:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776582.747858:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776582.747863:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776582.747867:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776582.747870:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776582.747874:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776582.747877:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776582.747881:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776582.747954:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776582.747958:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776582.747961:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776583.259870:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312970997, at 4312971008 00000100:00000010:0.0:1775776583.259885:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000040:1.0:1775776583.259887:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312970997, at 4312971008 00000100:00000010:1.0:1775776583.259896:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:3.0:1775776583.707940:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776583.707947:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776583.707948:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776583.707963:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776583.707966:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776583.707969:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776583.707972:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776583.707974:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776583.707977:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776583.771897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776583.771903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776583.771905:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776583.771910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776583.771913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776583.771917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776583.771920:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776583.771922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776583.771931:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776583.772103:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776583.772107:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776583.772110:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776584.731892:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776584.731900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776584.731902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776584.731907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776584.731910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776584.731914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776584.731916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776584.731918:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776584.731921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776584.795885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776584.795889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776584.795892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776584.795896:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776584.795900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776584.795903:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776584.795906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776584.795908:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776584.795917:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776584.795926:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776584.795930:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776584.795934:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776585.755902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776585.755906:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776585.755908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776585.755912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776585.755915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776585.755918:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776585.755920:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776585.755922:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776585.755924:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776585.819885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776585.819890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776585.819893:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776585.819897:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776585.819903:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776585.819906:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776585.819916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776585.819918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776585.819920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776585.819962:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776585.819965:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776585.819967:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776586.779920:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776586.779928:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776586.779930:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776586.779935:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776586.779939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776586.779955:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776586.779958:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776586.779960:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776586.779963:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776586.844008:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776586.844036:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776586.844039:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776586.844044:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776586.844048:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776586.844052:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776586.844054:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776586.844057:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776586.844059:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776586.844624:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775776586.844628:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775776586.844631:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776587.099893:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776587.099901:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776587.099903:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776587.099907:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776587.099910:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776587.099917:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18302 00000400:00000200:1.0:1775776587.099925:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776587.099929:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776587.099931:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776587.099934:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.099936:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.099938:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.099940:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776587.099943:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776587.099944:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776587.099946:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776587.099947:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776587.099949:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776587.099950:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776587.099951:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.099953:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.099954:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.099956:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776587.099958:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776587.099960:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776587.099961:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776587.099963:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776587.099965:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776587.099967:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.099969:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776587.099970:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776587.099979:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776587.099982:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776587.099986:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776587.099987:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.099990:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776587.099992:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776587.099997:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776587.100006:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775776587.100013:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776587.100022:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099769472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776587.100032:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776587.100036:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776587.100039:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099769472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776587.100062:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776587.100065:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776587.100068:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776587.100070:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776587.100071:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776587.100074:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776587.100076:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18302 00000400:00000200:1.0:1775776587.100080:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776587.100083:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776587.100085:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776587.100088:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100090:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.100093:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100095:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776587.100097:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776587.100098:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776587.100100:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776587.100102:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776587.100104:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776587.100106:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776587.100109:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100111:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.100113:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100115:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776587.100117:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776587.100119:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100121:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776587.100122:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776587.100126:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776587.100128:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100130:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776587.100135:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776587.100139:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100140:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776587.100145:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776587.100148:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776587.100150:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776587.100152:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100155:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776587.100157:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776587.100160:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776587.100165:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776587.100169:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776587.100177:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099769600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776587.100197:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776587.100202:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776587.100208:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099769600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776587.100223:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776587.100228:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776587.100233:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776587.100236:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776587.100238:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776587.100241:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776587.100243:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18302 00000400:00000200:1.0:1775776587.100250:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776587.100252:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776587.100254:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776587.100257:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100258:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.100260:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100261:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776587.100266:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776587.100267:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776587.100270:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776587.100272:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776587.100275:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776587.100276:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776587.100280:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100283:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.100284:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100286:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776587.100287:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776587.100288:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100289:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776587.100317:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776587.100320:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776587.100323:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100324:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776587.100327:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776587.100329:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100331:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776587.100336:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776587.100339:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776587.100342:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776587.100343:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100344:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776587.100346:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776587.100349:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776587.100356:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776587.100360:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776587.100368:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099769728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776587.100384:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776587.100388:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776587.100394:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099769728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776587.100408:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776587.100412:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776587.100419:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776587.100422:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776587.100425:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776587.100430:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776587.100435:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18302 00000400:00000200:1.0:1775776587.100442:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776587.100447:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776587.100450:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776587.100454:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100456:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.100459:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100461:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776587.100463:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776587.100465:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000800:00000200:2.1:1775776587.100466:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000001:1.0:1775776587.100467:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776587.100470:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776587.100473:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000800:00000200:2.1:1775776587.100473:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775776587.100476:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776587.100481:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100483:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.100486:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100489:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776587.100491:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776587.100496:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100498:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776587.100499:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776587.100501:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776587.100504:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100505:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776587.100508:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776587.100512:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100514:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776587.100517:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775776587.100521:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776587.100523:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776587.100525:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100527:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776587.100528:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776587.100530:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776587.100533:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776587.100536:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776587.100544:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099769856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776587.100554:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776587.100557:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776587.100563:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099769856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776587.100581:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776587.100583:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776587.100585:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776587.100586:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776587.100588:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776587.100594:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776587.100596:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18302 00000400:00000200:1.0:1775776587.100601:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776587.100603:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776587.100605:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776587.100607:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100609:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.100612:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100614:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776587.100616:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776587.100618:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776587.100621:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776587.100623:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776587.100626:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776587.100630:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776587.100634:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100637:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776587.100641:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100643:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776587.100647:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776587.100652:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776587.100656:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776587.100660:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776587.100665:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776587.100668:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100670:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776587.100672:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776587.100675:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100678:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776587.100683:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776587.100688:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776587.100693:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776587.100696:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776587.100699:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776587.100701:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776587.100704:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776587.100710:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776587.100715:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776587.100724:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099769984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776587.100747:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776587.100750:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776587.100757:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099769984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776587.100775:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776587.100807:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18312) 00000800:00000200:2.1:1775776587.101058:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776587.101060:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776587.101333:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776587.101336:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776587.101588:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776587.101590:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776587.101865:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776587.101867:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1775776587.803869:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776587.803874:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776587.803876:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776587.803880:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776587.803882:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776587.803885:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776587.803887:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776587.803889:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776587.803890:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776587.867932:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776587.867941:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776587.867943:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776587.867951:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776587.867957:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776587.867961:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776587.867967:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776587.867970:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776587.867977:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776587.868029:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776587.868032:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776587.868035:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776588.380377:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312976117, at 4312976128 00000100:00000010:1.0:1775776588.380396:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000040:0.0:1775776588.381420:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312976117, at 4312976129 00000100:00000010:0.0:1775776588.381429:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:3.0:1775776588.827927:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776588.827935:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776588.827937:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776588.827944:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776588.827949:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776588.827953:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776588.827956:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776588.827960:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776588.827963:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776588.891942:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776588.891950:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776588.891952:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776588.891959:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776588.891964:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776588.891969:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776588.891972:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776588.891975:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776588.891978:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776588.892537:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776588.892542:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776588.892545:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776589.851889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776589.851895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776589.851896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776589.851900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776589.851903:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776589.851905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776589.851907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776589.851909:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776589.851910:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776589.915899:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776589.915905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776589.915907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776589.915912:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776589.915916:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776589.915919:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776589.915922:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776589.915924:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776589.915927:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776589.916212:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775776589.916216:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775776589.916219:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776590.875943:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776590.875952:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776590.875954:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776590.875959:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776590.875970:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776590.875974:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776590.875976:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776590.875983:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776590.875985:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776590.939878:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776590.939886:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776590.939889:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776590.939895:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776590.939900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776590.939904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776590.939909:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776590.939914:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776590.939918:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776590.939959:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776590.939962:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776590.939964:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776591.899882:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776591.899894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776591.899896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776591.899901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776591.899905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776591.899908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776591.899910:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776591.899913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776591.899915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776591.963897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776591.963904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776591.963908:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776591.963914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776591.963919:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776591.963923:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776591.963927:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776591.963931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776591.963935:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776591.964073:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776591.964076:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776591.964080:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776592.220265:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776592.220274:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776592.220277:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776592.220281:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776592.220285:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776592.220291:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18307 00000400:00000200:3.0:1775776592.220300:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776592.220306:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776592.220309:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776592.220312:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220315:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.220317:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220319:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776592.220323:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776592.220325:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776592.220327:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776592.220330:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776592.220332:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776592.220334:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776592.220337:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220339:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.220340:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220342:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776592.220345:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776592.220347:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220350:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776592.220352:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776592.220355:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776592.220357:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220359:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776592.220361:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776592.220370:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776592.220373:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776592.220376:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776592.220378:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220381:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776592.220383:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776592.220387:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776592.220394:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776592.220400:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776592.220407:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099770112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776592.220416:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776592.220420:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776592.220425:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099770112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776592.220446:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776592.220449:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776592.220452:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776592.220454:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776592.220455:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776592.220458:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776592.220460:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18307 00000400:00000200:3.0:1775776592.220463:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776592.220466:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776592.220468:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776592.220471:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220474:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.220477:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220479:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776592.220481:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776592.220484:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776592.220487:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776592.220490:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776592.220492:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776592.220496:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776592.220501:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220505:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.220509:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220513:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776592.220516:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776592.220518:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220522:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776592.220525:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776592.220530:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776592.220533:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220535:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776592.220540:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776592.220544:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220548:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776592.220552:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776592.220556:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776592.220561:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776592.220563:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220567:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776592.220569:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776592.220572:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776592.220576:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776592.220579:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776592.220586:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099770240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776592.220598:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776592.220601:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776592.220606:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099770240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776592.220617:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776592.220622:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776592.220626:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776592.220629:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776592.220631:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776592.220635:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776592.220639:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18307 00000400:00000200:3.0:1775776592.220646:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776592.220652:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776592.220655:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776592.220660:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220664:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.220669:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220671:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776592.220673:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776592.220674:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776592.220675:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776592.220676:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776592.220678:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776592.220680:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776592.220684:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220686:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.220689:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220692:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776592.220696:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776592.220702:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220704:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776592.220709:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776592.220712:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776592.220715:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220716:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776592.220720:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776592.220723:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220726:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776592.220731:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776592.220735:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776592.220739:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776592.220740:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220743:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776592.220745:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776592.220747:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776592.220751:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776592.220756:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776592.220761:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099770368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776592.220773:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776592.220776:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776592.220803:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099770368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776592.220811:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776592.220814:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776592.220817:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776592.220818:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776592.220820:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776592.220822:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776592.220824:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18307 00000400:00000200:3.0:1775776592.220835:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776592.220838:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776592.220840:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776592.220847:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220850:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.220855:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220858:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776592.220860:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776592.220861:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776592.220863:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776592.220865:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776592.220868:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776592.220871:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776592.220874:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220879:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.220881:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220883:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776592.220886:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776592.220892:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776592.220895:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776592.220898:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776592.220901:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1775776592.220901:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:3.0:1775776592.220904:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220907:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.1:1775776592.220907:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000008:00100000:3.0:1775776592.220909:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776592.220913:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220917:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776592.220922:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776592.220925:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776592.220928:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776592.220929:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.220933:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776592.220935:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776592.220938:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776592.220944:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776592.220948:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776592.220956:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099770496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776592.220964:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776592.220966:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776592.220970:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099770496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776592.220988:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776592.220992:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776592.220996:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776592.220999:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776592.221001:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776592.221004:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776592.221008:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18307 00000400:00000200:3.0:1775776592.221016:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776592.221021:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776592.221025:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776592.221030:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.221032:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.221035:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.221038:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776592.221041:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776592.221043:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776592.221045:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776592.221047:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776592.221049:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776592.221053:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776592.221056:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.221060:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776592.221062:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776592.221064:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776592.221068:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776592.221071:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776592.221076:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776592.221078:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776592.221081:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776592.221084:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.221087:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776592.221089:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776592.221093:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.221095:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776592.221100:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776592.221103:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776592.221106:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776592.221109:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776592.221112:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776592.221114:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776592.221117:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776592.221121:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776592.221125:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776592.221130:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099770624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776592.221144:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776592.221147:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776592.221154:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099770624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776592.221173:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776592.221179:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18317) 00000800:00000200:2.1:1775776592.221372:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776592.221374:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776592.221608:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776592.221609:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776592.221877:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776592.221879:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776592.222131:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776592.222146:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1775776592.924273:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776592.924279:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776592.924280:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776592.924285:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776592.924287:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776592.924290:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776592.924292:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776592.924294:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776592.924296:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776592.987892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776592.987899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776592.987901:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776592.987905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776592.987908:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776592.987912:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776592.987915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776592.987918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776592.987923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776592.987995:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775776592.987999:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775776592.988002:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776593.499894:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312981238, at 4312981248 00000100:00000010:0.0:1775776593.499908:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000040:1.0:1775776593.501109:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312981238, at 4312981249 00000100:00000010:1.0:1775776593.501117:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775776593.948285:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776593.948292:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776593.948294:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776593.948300:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776593.948304:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776593.948308:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776593.948310:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776593.948313:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776593.948315:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776594.011929:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776594.011937:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776594.011938:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776594.011944:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776594.011947:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776594.011951:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776594.011953:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776594.011955:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776594.011958:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776594.013497:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776594.013501:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776594.013504:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776594.971931:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776594.971938:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776594.971940:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776594.971945:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776594.971948:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776594.971952:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776594.971966:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776594.971968:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776594.971971:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776595.035933:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776595.035940:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776595.035942:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776595.035948:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776595.035953:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776595.035958:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776595.035967:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776595.035970:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776595.035974:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776595.036027:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776595.036031:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776595.036034:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776595.995926:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776595.995934:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776595.995935:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776595.995941:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776595.995945:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776595.995949:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776595.995952:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776595.995954:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776595.995958:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776596.059857:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776596.059863:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776596.059864:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776596.059869:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776596.059872:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776596.059875:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776596.059877:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776596.059880:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776596.059882:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776596.060111:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775776596.060114:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775776596.060118:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776597.019932:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776597.019940:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776597.019941:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776597.019946:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776597.019949:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776597.019952:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776597.019955:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776597.019957:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776597.019959:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776597.083892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776597.083900:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776597.083903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776597.083908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:0.0:1775776597.083913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776597.083918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776597.083922:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776597.083926:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776597.083931:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776597.084007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776597.084011:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776597.084013:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776597.340178:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776597.340188:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776597.340191:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776597.340195:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776597.340199:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776597.340205:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18312 00000400:00000200:1.0:1775776597.340214:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776597.340220:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776597.340223:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776597.340226:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340229:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340231:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340234:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776597.340237:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776597.340239:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776597.340242:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776597.340244:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776597.340247:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776597.340249:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776597.340251:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340253:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340255:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340257:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776597.340260:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776597.340263:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340266:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776597.340268:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776597.340271:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776597.340274:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340276:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776597.340278:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776597.340288:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776597.340291:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776597.340295:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776597.340296:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340299:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776597.340302:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776597.340305:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776597.340312:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776597.340318:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776597.340326:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099770752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776597.340336:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776597.340341:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776597.340345:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099770752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776597.340367:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776597.340371:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776597.340374:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776597.340375:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776597.340377:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776597.340379:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776597.340381:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18312 00000400:00000200:1.0:1775776597.340385:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776597.340387:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776597.340389:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776597.340393:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340395:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340397:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340400:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776597.340403:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776597.340404:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776597.340405:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776597.340406:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776597.340409:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776597.340413:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776597.340417:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340419:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340422:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340425:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776597.340428:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776597.340433:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340437:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776597.340441:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776597.340447:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776597.340452:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340454:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776597.340459:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776597.340465:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340468:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776597.340474:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776597.340477:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776597.340481:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776597.340483:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340486:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776597.340487:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776597.340490:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776597.340495:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776597.340500:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776597.340507:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099770880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776597.340518:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776597.340521:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776597.340525:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099770880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776597.340541:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776597.340545:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776597.340553:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776597.340556:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776597.340559:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776597.340563:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776597.340567:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18312 00000400:00000200:1.0:1775776597.340572:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776597.340576:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776597.340579:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776597.340583:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340587:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340591:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340594:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776597.340597:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776597.340598:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776597.340601:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776597.340603:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776597.340606:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776597.340609:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776597.340614:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340617:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340621:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340625:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776597.340629:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776597.340635:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340640:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776597.340643:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776597.340645:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776597.340648:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340652:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776597.340654:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776597.340659:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340662:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776597.340665:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776597.340669:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776597.340672:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776597.340673:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340675:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776597.340676:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776597.340678:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776597.340681:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776597.340684:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776597.340688:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099771008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776597.340704:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776597.340707:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776597.340714:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099771008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776597.340725:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776597.340731:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776597.340732:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00080000:1.0:1775776597.340735:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776597.340740:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776597.340742:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776597.340745:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776597.340750:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776597.340753:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18312 00000400:00000200:1.0:1775776597.340757:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776597.340763:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776597.340767:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776597.340770:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340806:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340810:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340812:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776597.340815:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776597.340816:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776597.340818:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776597.340820:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776597.340822:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776597.340825:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776597.340830:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340834:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340839:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340843:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776597.340847:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776597.340853:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340859:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776597.340864:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776597.340867:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776597.340871:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340873:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776597.340876:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776597.340880:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340884:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776597.340890:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776597.340893:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776597.340896:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776597.340898:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.340900:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776597.340902:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776597.340904:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776597.340910:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776597.340914:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776597.340921:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099771136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776597.340932:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776597.340935:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776597.340941:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099771136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776597.340956:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776597.340961:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776597.340965:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776597.340966:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776597.340967:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776597.340969:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776597.340972:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18312 00000400:00000200:1.0:1775776597.340979:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776597.340984:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776597.340987:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776597.340992:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.340995:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.340999:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.341002:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776597.341006:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776597.341008:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776597.341011:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776597.341014:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776597.341017:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776597.341021:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776597.341028:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.341033:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776597.341037:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776597.341040:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776597.341043:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776597.341048:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776597.341052:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776597.341054:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776597.341057:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776597.341062:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.341066:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776597.341069:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776597.341073:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.341075:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776597.341080:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776597.341083:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776597.341085:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776597.341088:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776597.341091:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776597.341094:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776597.341097:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776597.341101:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776597.341105:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776597.341123:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099771264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776597.341139:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776597.341144:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776597.341152:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099771264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776597.341165:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776597.341170:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18322) 00000800:00000200:2.1:1775776597.341335:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776597.341337:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776597.341580:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776597.341583:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776597.341815:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776597.341817:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776597.342021:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776597.342023:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775776598.043945:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776598.043952:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776598.043954:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776598.043959:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776598.043962:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776598.043966:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776598.043969:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776598.043971:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776598.043974:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776598.107991:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776598.107994:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776598.107997:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776598.108088:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776598.108092:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776598.108094:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776598.108098:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776598.108128:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776598.108131:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776598.108134:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776598.108144:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776598.108150:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776598.619882:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312986358, at 4312986368 00000100:00000010:1.0:1775776598.619896:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000040:0.0:1775776598.620013:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312986358, at 4312986368 00000100:00000010:0.0:1775776598.620020:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000001:3.0:1775776599.067888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776599.067894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776599.067896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776599.067900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776599.067904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776599.067908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776599.067911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776599.067913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776599.067916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776599.131886:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776599.131893:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776599.131895:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776599.131901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776599.131907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776599.131911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776599.131917:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776599.131920:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776599.131924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776599.131992:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776599.131995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776599.131998:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776600.091899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776600.091907:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776600.091909:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776600.091923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776600.091928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776600.091932:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776600.091934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776600.091936:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776600.091939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.155960:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776600.155967:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776600.155969:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776600.155975:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776600.155990:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776600.155994:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776600.155998:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776600.156001:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776600.156004:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776600.157011:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776600.157015:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776600.157018:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776601.116184:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776601.116191:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776601.116193:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776601.116198:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776601.116202:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776601.116206:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776601.116208:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776601.116211:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776601.116213:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776601.179834:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776601.179842:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776601.179844:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776601.179850:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776601.179854:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776601.179858:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776601.179860:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776601.179863:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776601.179865:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776601.180158:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776601.180162:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776601.180164:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776602.139952:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776602.139960:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776602.139961:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776602.139966:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776602.139970:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776602.139974:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776602.139977:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776602.139979:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776602.139981:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.203844:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776602.203853:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776602.203855:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776602.203860:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776602.203863:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776602.203866:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776602.203869:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776602.203871:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776602.203874:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776602.204740:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776602.204744:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776602.204752:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776602.459955:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776602.459968:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776602.459971:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776602.459976:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776602.459981:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776602.459991:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18317 00000400:00000200:3.0:1775776602.460003:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776602.460010:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776602.460014:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776602.460017:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460021:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460039:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460041:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776602.460046:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776602.460049:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776602.460052:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776602.460056:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776602.460058:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776602.460061:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776602.460063:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460065:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460067:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460069:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776602.460072:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776602.460076:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460078:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776602.460082:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776602.460086:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776602.460089:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460092:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776602.460095:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776602.460108:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776602.460112:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776602.460117:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776602.460119:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460123:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776602.460126:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776602.460140:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776602.460150:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776602.460158:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776602.460169:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099771392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776602.460180:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776602.460185:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776602.460190:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099771392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776602.460221:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776602.460225:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776602.460228:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776602.460229:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776602.460231:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776602.460234:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776602.460236:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18317 00000400:00000200:3.0:1775776602.460239:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776602.460242:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776602.460245:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776602.460248:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460250:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460252:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460254:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776602.460256:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776602.460257:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776602.460259:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776602.460261:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776602.460263:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776602.460265:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776602.460268:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460270:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460272:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460273:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776602.460275:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776602.460278:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460281:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776602.460283:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776602.460290:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776602.460292:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460294:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776602.460298:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776602.460301:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460304:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776602.460307:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776602.460309:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776602.460312:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776602.460313:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460314:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776602.460315:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776602.460317:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776602.460321:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776602.460324:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776602.460329:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099771520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776602.460337:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776602.460339:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776602.460342:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099771520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776602.460350:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776602.460352:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776602.460356:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776602.460358:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776602.460359:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776602.460362:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776602.460364:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18317 00000400:00000200:3.0:1775776602.460367:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776602.460370:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776602.460372:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776602.460375:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460377:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460379:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460381:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776602.460383:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776602.460384:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776602.460385:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776602.460387:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776602.460389:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776602.460391:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776602.460394:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460396:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460398:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460400:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776602.460402:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776602.460405:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460407:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776602.460408:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776602.460411:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776602.460413:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460415:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776602.460417:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776602.460419:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460421:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776602.460424:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776602.460427:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776602.460429:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776602.460430:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460432:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776602.460434:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776602.460435:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776602.460439:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775776602.460442:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776602.460446:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099771648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776602.460453:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776602.460455:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776602.460458:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099771648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776602.460465:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776602.460467:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776602.460471:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776602.460472:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776602.460473:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776602.460476:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776602.460478:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18317 00000400:00000200:3.0:1775776602.460481:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776602.460484:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776602.460486:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776602.460489:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460491:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460492:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460494:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776602.460496:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776602.460497:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776602.460498:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776602.460500:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776602.460502:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776602.460504:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776602.460507:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460509:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460511:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460513:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776602.460515:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776602.460517:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460520:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776602.460521:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776602.460524:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776602.460527:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460529:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776602.460531:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776602.460533:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460535:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776602.460538:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776602.460540:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776602.460543:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776602.460544:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460546:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776602.460547:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776602.460549:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776602.460553:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776602.460556:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776602.460560:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099771776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776602.460567:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776602.460569:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776602.460572:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099771776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776602.460578:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776602.460580:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776602.460584:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776602.460585:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776602.460586:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776602.460588:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776602.460590:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18317 00000400:00000200:3.0:1775776602.460593:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776602.460596:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776602.460598:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776602.460600:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460602:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460604:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460606:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776602.460608:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776602.460609:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776602.460611:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776602.460612:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776602.460614:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776602.460616:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776602.460630:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460631:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776602.460633:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460639:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776602.460641:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776602.460643:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776602.460645:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776602.460647:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776602.460649:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776602.460651:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460653:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776602.460655:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776602.460657:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460659:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776602.460662:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776602.460664:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776602.460667:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776602.460668:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776602.460670:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776602.460671:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776602.460672:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776602.460677:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776602.460681:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776602.460684:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099771904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776602.460691:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776602.460692:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776602.460695:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099771904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776602.460702:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776602.460707:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18327) 00000800:00000200:2.1:1775776602.461004:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776602.461008:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776602.461354:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776602.461357:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776602.461607:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776602.461609:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776602.461884:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776602.461886:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1775776602.462146:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776602.462148:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1775776603.163994:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776603.164008:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776603.164009:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776603.164028:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776603.164032:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776603.164044:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776603.164047:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776603.164049:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776603.164052:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776603.228227:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776603.228231:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776603.228233:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776603.228237:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776603.228241:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776603.228244:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776603.228246:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776603.228248:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776603.228251:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776603.230089:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776603.230131:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776603.230136:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776603.739898:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312991478, at 4312991488 00000100:00000010:0.0:1775776603.739959:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000040:1.0:1775776603.740065:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312991478, at 4312991488 00000100:00000010:1.0:1775776603.740072:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:3.0:1775776604.187897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776604.187905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776604.187907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776604.187912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776604.187916:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776604.187920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776604.187922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776604.187924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776604.187926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776604.251873:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776604.251882:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776604.251885:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776604.251893:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776604.251898:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776604.251903:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776604.251908:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776604.251911:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776604.251916:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776604.251986:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776604.251990:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776604.252005:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776605.211864:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776605.211871:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776605.211873:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776605.211878:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776605.211883:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776605.211886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776605.211889:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776605.211892:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776605.211895:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776605.275841:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776605.275847:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776605.275849:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776605.275854:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776605.275857:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776605.275860:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776605.275863:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776605.275865:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776605.275868:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776605.277679:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776605.277684:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776605.277688:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776606.235891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776606.235897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776606.235899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776606.235904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776606.235907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776606.235910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776606.235912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776606.235914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776606.235916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776606.299878:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776606.299884:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776606.299885:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776606.299891:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776606.299894:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776606.299897:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776606.299900:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776606.299902:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776606.299905:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776606.299977:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776606.299981:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776606.299985:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776607.259881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776607.259888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776607.259890:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776607.259894:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776607.259898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776607.259901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776607.259903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776607.259906:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776607.259908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.323905:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776607.323912:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776607.323970:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776607.323975:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776607.323979:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776607.323983:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776607.323985:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776607.323988:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776607.323990:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776607.324212:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776607.324216:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776607.324219:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776607.579894:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776607.579904:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776607.579942:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776607.579946:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776607.579950:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776607.579957:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18322 00000400:00000200:3.0:1775776607.579966:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776607.579972:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776607.579975:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776607.579978:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.579982:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.579984:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.579987:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776607.579991:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776607.579992:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776607.579995:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776607.580010:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776607.580012:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776607.580013:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776607.580016:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580017:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580018:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580019:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776607.580021:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776607.580024:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580026:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776607.580029:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776607.580032:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776607.580034:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580036:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776607.580039:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776607.580048:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000020:00000040:3.0:1775776607.580052:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776607.580056:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776607.580057:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580060:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776607.580062:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776607.580066:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776607.580073:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776607.580080:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776607.580089:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099772032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776607.580100:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776607.580104:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776607.580109:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099772032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776607.580130:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776607.580133:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776607.580136:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776607.580137:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776607.580139:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776607.580141:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776607.580143:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18322 00000400:00000200:3.0:1775776607.580147:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776607.580150:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776607.580153:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776607.580156:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580158:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580161:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580163:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776607.580165:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776607.580167:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776607.580168:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776607.580171:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776607.580173:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776607.580176:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776607.580180:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580184:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580188:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580190:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776607.580195:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776607.580201:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580207:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776607.580211:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776607.580218:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776607.580232:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580236:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776607.580242:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776607.580247:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580249:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776607.580253:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775776607.580255:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776607.580258:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776607.580259:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580261:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776607.580262:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776607.580264:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776607.580268:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776607.580271:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776607.580278:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099772160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776607.580288:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776607.580291:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776607.580294:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099772160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776607.580308:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776607.580310:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776607.580314:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776607.580317:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776607.580318:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776607.580323:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776607.580327:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18322 00000400:00000200:3.0:1775776607.580333:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776607.580338:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776607.580341:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776607.580347:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580349:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580351:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580355:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776607.580360:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776607.580363:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776607.580365:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776607.580367:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776607.580371:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776607.580373:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776607.580376:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580378:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580379:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580381:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776607.580384:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776607.580390:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580393:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776607.580395:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776607.580399:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776607.580403:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580405:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776607.580408:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776607.580412:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580415:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776607.580421:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776607.580427:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776607.580432:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776607.580434:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580437:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776607.580440:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776607.580443:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776607.580450:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776607.580454:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776607.580463:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099772288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776607.580477:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776607.580481:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776607.580486:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099772288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776607.580500:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776607.580504:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1775776607.580510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:3.0:1775776607.580511:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776607.580513:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776607.580515:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1775776607.580517:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1775776607.580520:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776607.580525:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18322 00000400:00000200:3.0:1775776607.580532:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776607.580538:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776607.580542:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776607.580548:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580551:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580553:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580556:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776607.580558:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776607.580560:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776607.580562:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776607.580563:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776607.580566:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776607.580567:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776607.580571:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580575:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580580:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580584:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776607.580587:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776607.580593:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580598:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776607.580603:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776607.580607:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776607.580613:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580616:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776607.580618:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776607.580623:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580627:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776607.580632:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776607.580636:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776607.580640:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776607.580642:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580645:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776607.580646:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776607.580649:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776607.580652:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776607.580657:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776607.580664:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099772416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776607.580678:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776607.580681:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776607.580688:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099772416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776607.580711:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776607.580715:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776607.580721:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776607.580723:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776607.580732:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776607.580737:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776607.580742:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18322 00000400:00000200:3.0:1775776607.580751:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776607.580759:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776607.580762:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776607.580769:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580774:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580812:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580817:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776607.580823:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776607.580825:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776607.580828:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776607.580830:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776607.580832:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776607.580834:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776607.580837:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580841:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776607.580846:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580851:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776607.580855:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776607.580861:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776607.580866:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776607.580870:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776607.580874:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776607.580879:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580883:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776607.580886:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776607.580891:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580895:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776607.580900:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776607.580903:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776607.580906:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776607.580908:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776607.580911:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776607.580913:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776607.580915:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776607.580919:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776607.580923:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776607.580931:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099772544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776607.580962:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776607.580966:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776607.580973:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099772544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776607.580987:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776607.580992:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18333) 00000800:00000200:2.1:1775776607.581145:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776607.581148:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776607.581408:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776607.581410:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776607.581674:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776607.581677:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776607.581970:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776607.581972:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1775776608.283986:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776608.284005:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776608.284007:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776608.284013:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776608.284016:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776608.284019:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776608.284022:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776608.284024:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776608.284027:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776608.347966:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776608.347975:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776608.347977:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776608.347982:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776608.347986:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776608.347989:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776608.347993:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776608.347997:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776608.348001:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776608.348091:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776608.348104:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776608.348107:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776608.859901:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4312996597, at 4312996608 00000100:00000010:0.0:1775776608.859941:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000040:1.0:1775776608.860042:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4312996597, at 4312996608 00000100:00000010:1.0:1775776608.860048:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:3.0:1775776609.307897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776609.307917:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776609.307920:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776609.307925:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776609.307929:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776609.307932:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776609.307936:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776609.307938:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776609.307940:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776609.371875:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776609.371881:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776609.371883:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776609.371890:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776609.371895:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776609.371899:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776609.371942:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776609.371945:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776609.371949:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776609.371988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776609.371992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776609.371994:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776610.331906:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776610.331913:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776610.331915:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776610.331919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776610.331923:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776610.331927:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776610.331929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776610.331932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776610.331934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776610.395910:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776610.395917:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776610.395921:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776610.395928:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776610.395933:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776610.395939:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776610.395942:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776610.395947:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776610.395951:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776610.396070:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776610.396073:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776610.396076:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776611.356066:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776611.356073:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776611.356075:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776611.356080:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776611.356084:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776611.356087:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776611.356090:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776611.356092:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776611.356094:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776611.419901:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776611.419909:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776611.419912:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776611.419920:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776611.419926:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776611.419932:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776611.419937:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776611.419939:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776611.419941:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776611.420029:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776611.420032:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776611.420035:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776612.379943:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776612.379951:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776612.379953:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776612.379959:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776612.379963:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776612.379967:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.379970:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776612.379973:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.379975:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.443887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776612.443893:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776612.443895:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776612.443901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776612.443907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776612.443911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.443914:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776612.443916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.443919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776612.443963:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776612.443966:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776612.443968:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776612.700118:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776612.700122:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776612.700125:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776612.700130:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700138:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776612.700140:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.700143:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776612.700146:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776612.700148:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776612.700151:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099772672:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776612.700155:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776612.700158:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776612.700160:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.700163:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776612.700167:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776612.700169:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776612.700175:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776612.700186:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776612.700192:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776612.700199:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099772672, portal 25 00000100:00000001:3.0:1775776612.700201:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776612.700203:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288712448 : -114841420839168 : ffff978d65e7c700) 00000100:00000040:3.0:1775776612.700207:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700215:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776612.700218:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776612.700220:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776612.700224:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099772672, offset 0 00000400:00000200:3.0:1775776612.700234:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776612.700241:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.700252:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776612.700257:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776612.700261:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776612.700265:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776612.700267:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776612.700271:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4267:4267:256:4294967295] 192.168.201.139@tcp LPNI seq info [4267:4267:8:4294967295] 00000400:00000200:3.0:1775776612.700276:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.700286:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776612.700292:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.700296:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776612.700301:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776612.700305:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776612.700310:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776612.700314:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776612.700318:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.700320:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776612.700353:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776612.700357:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776612.700362:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776612.700364:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776612.700368:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776612.700372:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700381:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.700384:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700397:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.700398:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776612.700409:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776612.700416:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776612.700418:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313001717 00000100:00000010:3.0:1775776612.700423:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344180. 00000100:00000001:3.0:1775776612.700428:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776612.700443:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313001717 00000100:00000010:3.0:1775776612.700446:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344900. 00000100:00000001:3.0:1775776612.700449:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776612.700452:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776612.700455:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.700458:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776612.700461:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776612/real 1775776612] req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776612.700471:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776612.700473:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776612.700475:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776612.700478:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776612.700480:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776612.700483:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776612.700486:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776612.700495:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700502:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776612.700503:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700505:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.700507:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776612.700511:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700518:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776612.700522:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700529:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776612.700532:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700533:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.700535:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700537:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776612.700539:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776612.700541:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776612.700545:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776612.700547:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776612.700551:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776612.700558:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099772672:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776612.700563:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700565:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.700567:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c700 x1862033099772672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700574:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776612.700576:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776612.700579:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776612.700581:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776612.700583:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776612.700585:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776612.700586:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776612.700588:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:3.0:1775776612.700590:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776612.700592:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776612.700594:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c700. 00000100:00000001:3.0:1775776612.700596:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776612.700596:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700598:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700603:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776612.700604:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.700605:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776612.700606:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776612.700607:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776612.700609:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700614:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776612.700614:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.700616:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776612.700617:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776612.700618:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776612.700620:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099772800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776612.700622:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776612.700623:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776612.700624:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.700625:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776612.700627:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776612.700628:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776612.700630:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776612.700632:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776612.700634:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776612.700636:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099772800, portal 10 00000100:00000001:3.0:1775776612.700637:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776612.700638:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288722304 : -114841420829312 : ffff978d65e7ed80) 00000100:00000040:3.0:1775776612.700642:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700646:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776612.700648:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776612.700649:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776612.700651:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099772800, offset 0 00000400:00000200:3.0:1775776612.700653:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776612.700655:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.700658:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776612.700659:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776612.700661:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776612.700663:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776612.700664:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776612.700666:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4268:4268:256:4294967295] 192.168.201.139@tcp LPNI seq info [4268:4268:8:4294967295] 00000400:00000200:3.0:1775776612.700668:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.700673:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776612.700677:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.700680:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776612.700684:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776612.700686:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776612.700690:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776612.700692:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776612.700708:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.700711:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776612.700713:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776612.700717:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776612.700720:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776612.700722:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776612.700724:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776612.700728:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700735:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.700738:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700744:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.700745:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776612.700748:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776612.700751:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776612.700753:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776612.700754:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313000452 00000100:00000001:3.0:1775776612.700757:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313001717 00000100:00000001:3.0:1775776612.700759:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313000452 00000100:00000001:3.0:1775776612.700761:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313001717 00000100:00000001:3.0:1775776612.700763:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776612.700765:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.700767:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776612.700770:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776612/real 1775776612] req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776612.700801:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776612.700802:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776612.700804:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776612.700807:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776612.700808:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776612.700811:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776612.700814:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776612.700821:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700825:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776612.700826:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700828:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.700828:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776612.700830:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700834:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776612.700837:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700841:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776612.700843:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700844:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.700862:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.700864:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776612.700865:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776612.700868:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776612.700871:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776612.700873:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776612.700876:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776612.700883:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099772800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776612.700887:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776612.700890:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700896:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776612.700897:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.700899:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776612.700901:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776612.700902:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776612.700905:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099772928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776612.700908:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776612.700910:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776612.700911:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.700913:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776612.700916:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776612.700918:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776612.700922:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776612.700925:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776612.700928:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776612.700931:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099772928, portal 10 00000100:00000001:3.0:1775776612.700933:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776612.700934:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724992 : -114841420826624 : ffff978d65e7f800) 00000100:00000040:3.0:1775776612.700939:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.700946:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776612.700948:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776612.700950:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776612.700952:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099772928, offset 0 00000400:00000200:3.0:1775776612.700956:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776612.700960:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.700964:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776612.700966:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776612.700968:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776612.700970:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776612.700972:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776612.700975:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4269:4269:256:4294967295] 192.168.201.139@tcp LPNI seq info [4269:4269:8:4294967295] 00000400:00000200:3.0:1775776612.700978:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.700983:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776612.700987:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.700990:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776612.700994:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776612.700996:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776612.701000:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776612.701002:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776612.701005:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.701007:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776612.701010:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776612.701013:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776612.701016:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776612.701018:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776612.701020:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776612.701024:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701031:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.701034:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701040:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.701041:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776612.701044:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776612.701047:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776612.701050:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776612.701051:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313000454 00000100:00000001:3.0:1775776612.701054:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313001717 00000100:00000001:3.0:1775776612.701056:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313000454 00000100:00000001:3.0:1775776612.701057:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313001717 00000100:00000001:3.0:1775776612.701059:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776612.701061:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.701063:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776612.701066:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776612/real 1775776612] req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776612.701073:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776612.701074:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776612.701076:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776612.701078:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776612.701079:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776612.701081:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776612.701084:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776612.701091:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701097:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776612.701099:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701100:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.701102:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776612.701105:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701111:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776612.701114:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701120:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776612.701122:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701124:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.701125:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701127:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776612.701128:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776612.701130:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776612.701133:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776612.701135:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776612.701138:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776612.701146:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099772928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776612.701149:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776612.701152:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701157:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776612.701158:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.701160:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776612.701162:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776612.701164:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776612.701167:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099773056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776612.701170:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776612.701172:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776612.701174:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.701175:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776612.701178:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776612.701180:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776612.701184:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776612.701187:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776612.701190:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776612.701193:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099773056, portal 4 00000100:00000001:3.0:1775776612.701195:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776612.701196:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288711552 : -114841420840064 : ffff978d65e7c380) 00000100:00000040:3.0:1775776612.701200:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701204:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776612.701205:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776612.701206:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776612.701208:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099773056, offset 0 00000400:00000200:3.0:1775776612.701210:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776612.701213:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.701215:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776612.701217:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776612.701218:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776612.701220:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776612.701221:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776612.701223:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4270:4270:256:4294967295] 192.168.201.139@tcp LPNI seq info [4270:4270:8:4294967295] 00000400:00000200:3.0:1775776612.701226:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.701229:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776612.701232:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.701234:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776612.701236:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776612.701238:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776612.701241:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776612.701243:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776612.701245:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.701246:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776612.701248:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776612.701250:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776612.701251:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776612.701252:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776612.701254:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776612.701256:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701261:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.701263:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701269:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.701270:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776612.701273:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776612.701275:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776612.701277:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776612.701279:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313000463 00000100:00000001:3.0:1775776612.701281:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313001717 00000100:00000001:3.0:1775776612.701283:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313000463 00000100:00000001:3.0:1775776612.701284:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313001717 00000100:00000001:3.0:1775776612.701286:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776612.701288:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.701290:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776612.701293:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776612/real 1775776612] req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776612.701301:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776612.701302:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776612.701304:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776612.701306:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776612.701307:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776612.701309:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776612.701312:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776612.701320:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701326:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776612.701327:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701328:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.701329:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776612.701332:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701338:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776612.701341:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701348:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776612.701350:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701351:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.701353:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701354:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776612.701356:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776612.701358:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776612.701361:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776612.701363:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776612.701367:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776612.701373:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099773056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776612.701376:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776612.701378:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701382:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776612.701383:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.701384:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776612.701385:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776612.701386:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776612.701388:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099773184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776612.701389:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776612.701390:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776612.701391:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776612.701392:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776612.701395:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776612.701396:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776612.701398:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776612.701400:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776612.701401:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776612.701403:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099773184, portal 4 00000100:00000001:3.0:1775776612.701405:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776612.701405:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288718720 : -114841420832896 : ffff978d65e7df80) 00000100:00000040:3.0:1775776612.701409:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701413:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776612.701414:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776612.701415:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776612.701417:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099773184, offset 0 00000400:00000200:3.0:1775776612.701420:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776612.701422:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.701425:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776612.701428:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776612.701430:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776612.701433:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776612.701435:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776612.701438:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4271:4271:256:4294967295] 192.168.201.139@tcp LPNI seq info [4271:4271:8:4294967295] 00000400:00000200:3.0:1775776612.701441:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776612.701447:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776612.701451:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.701454:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776612.701457:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776612.701460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775776612.701464:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776612.701466:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1775776612.701468:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776612.701469:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776612.701471:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776612.701473:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776612.701475:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776612.701476:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776612.701477:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776612.701480:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701487:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.701490:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776637 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701495:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.701496:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776612.701499:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776612.701502:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776612.701504:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776612.701506:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313000454 00000100:00000001:3.0:1775776612.701509:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313001717 00000100:00000001:3.0:1775776612.701510:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313000454 00000100:00000001:3.0:1775776612.701512:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313001717 00000100:00000001:3.0:1775776612.701514:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776612.701516:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.701518:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776612.701521:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776612/real 1775776612] req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776612.701529:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776612.701530:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776612.701532:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776612.701534:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776612.701536:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776612.701538:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776612.701541:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776612.701546:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701550:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776612.701551:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701552:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.701552:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776612.701554:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701558:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776612.701560:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701564:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776612.701565:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701566:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776612.701567:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701568:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776612.701569:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776612.701571:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776612.701574:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776612.701576:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776612.701579:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776612.701585:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099773184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776612.701588:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701590:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.701593:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ed80 x1862033099772800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701598:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776612.701600:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776612.701602:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158800. 02000000:00000001:3.0:1775776612.701604:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776612.701606:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776612.701607:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776612.701609:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776612.701611:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507000. 02000000:00000001:3.0:1775776612.701613:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776612.701615:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776612.701616:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ed80. 00000100:00000001:3.0:1775776612.701620:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776612.701621:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701622:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.701625:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f800 x1862033099772928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701634:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776612.701635:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776612.701636:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158000. 02000000:00000001:3.0:1775776612.701637:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776612.701638:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776612.701638:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776612.701639:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776612.701640:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:3.0:1775776612.701641:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776612.701642:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776612.701642:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f800. 00000100:00000001:3.0:1775776612.701643:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776612.701644:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701645:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.701647:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099773056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701651:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776612.701651:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776612.701652:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115ac00. 02000000:00000001:3.0:1775776612.701653:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776612.701654:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776612.701654:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776612.701655:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776612.701656:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505400. 02000000:00000001:3.0:1775776612.701657:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776612.701657:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776612.701658:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:3.0:1775776612.701659:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776612.701659:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701660:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776612.701662:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7df80 x1862033099773184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776637 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776612.701665:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776612.701665:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776612.701667:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a000. 02000000:00000001:3.0:1775776612.701668:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776612.701670:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776612.701671:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776612.701672:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776612.701674:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504400. 02000000:00000001:3.0:1775776612.701676:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776612.701676:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776612.701678:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7df80. 00000100:00000001:3.0:1775776612.701680:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776612.701681:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701682:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776612.701686:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776612.701688:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776612.701690:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776612.701692:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776613.403910:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776613.403916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776613.403918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776613.403923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776613.403927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776613.403930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776613.403933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776613.403935:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776613.403938:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776613.467876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776613.467884:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776613.467887:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776613.467893:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776613.467899:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776613.467904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776613.467908:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776613.467913:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776613.467917:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776613.467968:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776613.467972:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776613.467976:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776613.979907:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313001717, at 4313001728 00000100:00000040:1.0:1775776613.979907:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313001717, at 4313001728 00000100:00000010:0.0:1775776613.979923:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344180. 00000100:00000010:1.0:1775776613.979924:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000100:00000001:3.0:1775776614.427951:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776614.427958:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776614.427960:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776614.427966:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776614.427970:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776614.427973:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776614.427976:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776614.427978:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776614.427981:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776614.491893:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776614.491899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776614.491903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776614.491909:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776614.491912:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776614.491915:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776614.491918:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776614.491919:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776614.491923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776614.491985:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776614.491988:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776614.491997:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776615.451951:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776615.451957:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776615.451959:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776615.451965:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776615.451969:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776615.451973:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776615.451975:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776615.451977:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776615.451980:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776615.515914:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776615.515919:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776615.515921:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776615.515926:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776615.515929:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776615.515933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776615.515935:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776615.515938:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776615.515941:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776615.515954:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776615.515958:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776615.515961:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776616.476888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776616.476897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776616.476899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776616.476904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776616.476908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776616.476911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776616.476914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776616.476917:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776616.476919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776616.539913:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776616.539925:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776616.539928:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776616.539933:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776616.539939:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776616.539944:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776616.539955:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776616.539958:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776616.539961:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776616.540015:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776616.540019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776616.540022:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776617.499892:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776617.499899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776617.499901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776617.499907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776617.499910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776617.499915:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776617.499917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776617.499920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776617.499922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.563863:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776617.563870:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776617.563873:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776617.563880:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776617.563885:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776617.563891:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776617.563894:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776617.563898:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776617.563901:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776617.563956:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776617.563958:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776617.563960:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776617.819972:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776617.819979:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776617.819981:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776617.819984:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776617.819987:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776617.819992:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18333 00000400:00000200:3.0:1775776617.820001:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776617.820006:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776617.820008:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776617.820010:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820012:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820013:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820015:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776617.820018:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776617.820019:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776617.820021:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776617.820023:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776617.820024:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776617.820025:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776617.820027:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820028:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820029:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820030:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776617.820032:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776617.820034:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820035:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776617.820037:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776617.820039:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776617.820041:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820042:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776617.820044:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776617.820051:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776617.820054:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776617.820056:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776617.820057:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820059:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776617.820061:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776617.820064:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776617.820069:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776617.820074:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776617.820080:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099773312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776617.820087:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776617.820090:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776617.820093:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099773312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776617.820111:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776617.820113:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776617.820115:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776617.820116:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776617.820117:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776617.820118:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776617.820120:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18333 00000400:00000200:3.0:1775776617.820122:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776617.820123:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776617.820125:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776617.820126:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820128:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820129:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820131:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776617.820132:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776617.820134:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776617.820135:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776617.820136:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776617.820137:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776617.820138:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776617.820140:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820141:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820143:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820145:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776617.820148:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776617.820150:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820153:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776617.820155:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776617.820159:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776617.820161:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820166:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776617.820172:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776617.820176:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820179:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776617.820183:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776617.820187:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776617.820193:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776617.820196:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820200:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776617.820203:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776617.820207:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776617.820211:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776617.820216:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776617.820223:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099773440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776617.820235:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776617.820238:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776617.820243:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099773440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776617.820267:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776617.820271:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776617.820278:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776617.820281:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776617.820285:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776617.820290:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776617.820296:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18333 00000400:00000200:3.0:1775776617.820302:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776617.820310:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776617.820315:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776617.820319:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820324:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820327:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820332:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776617.820337:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776617.820340:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776617.820345:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776617.820349:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776617.820354:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776617.820359:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776617.820365:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820370:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820376:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820380:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776617.820384:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776617.820390:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820392:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776617.820396:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776617.820400:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776617.820405:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820410:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776617.820412:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776617.820419:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820422:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776617.820422:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776617.820427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000010:3.0:1775776617.820428:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776617.820432:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776617.820439:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776617.820442:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820445:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776617.820446:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776617.820448:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776617.820452:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775776617.820456:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776617.820461:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099773568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776617.820468:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776617.820470:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776617.820473:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099773568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776617.820481:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776617.820483:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776617.820486:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776617.820487:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776617.820488:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776617.820490:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776617.820493:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18333 00000400:00000200:3.0:1775776617.820498:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776617.820503:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776617.820508:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776617.820513:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820519:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820523:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820527:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776617.820533:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776617.820545:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776617.820552:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776617.820556:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776617.820561:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776617.820565:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776617.820572:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820576:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820580:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820583:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776617.820587:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776617.820591:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820596:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776617.820600:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776617.820604:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776617.820607:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820610:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776617.820613:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776617.820618:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820620:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776617.820625:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776617.820629:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776617.820634:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776617.820636:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820641:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776617.820643:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776617.820645:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776617.820651:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776617.820657:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776617.820664:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099773696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776617.820675:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776617.820679:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776617.820688:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099773696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776617.820705:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776617.820710:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776617.820716:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776617.820719:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776617.820722:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776617.820726:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776617.820730:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18333 00000400:00000200:3.0:1775776617.820737:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776617.820742:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776617.820744:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776617.820748:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820750:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820754:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820773:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776617.820774:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776617.820775:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776617.820805:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776617.820807:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776617.820809:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776617.820811:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776617.820816:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820820:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776617.820825:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820829:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776617.820833:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776617.820839:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776617.820844:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776617.820850:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776617.820854:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776617.820859:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820871:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776617.820874:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776617.820880:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820884:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776617.820891:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776617.820896:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776617.820901:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776617.820904:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776617.820909:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776617.820912:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776617.820916:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776617.820923:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776617.820930:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776617.820937:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099773824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776617.820953:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776617.820956:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776617.820962:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099773824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776617.820977:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776617.820982:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18343) 00000800:00000200:2.1:1775776617.821101:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776617.821103:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776617.821326:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776617.821328:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776617.821552:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776617.821555:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776617.821894:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776617.821896:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1775776618.523903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776618.523908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776618.523909:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776618.523913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776618.523915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776618.523918:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776618.523920:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776618.523922:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776618.523923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776618.587893:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776618.587900:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776618.587904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776618.587909:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776618.587914:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776618.587918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776618.587934:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776618.587938:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776618.587941:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776618.587998:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776618.588002:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776618.588005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776619.099896:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313006837, at 4313006848 00000100:00000040:0.0:1775776619.099897:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313006837, at 4313006848 00000100:00000010:0.0:1775776619.099910:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000010:1.0:1775776619.099912:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000001:3.0:1775776619.547931:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776619.547938:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776619.547940:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776619.547946:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776619.547949:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776619.547954:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776619.547956:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776619.547958:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776619.547960:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776619.611905:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776619.611913:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776619.611915:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776619.611922:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776619.611930:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776619.611934:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776619.611937:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776619.611941:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776619.611945:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776619.612052:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776619.612055:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776619.612057:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776620.571904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776620.571910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776620.571913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776620.571918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776620.571922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776620.571926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776620.571929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776620.571932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776620.571934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776620.635934:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776620.635943:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776620.635946:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776620.635953:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776620.635958:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776620.635963:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776620.635967:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776620.635971:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776620.635976:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776620.636050:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776620.636055:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776620.636058:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776621.596007:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776621.596014:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776621.596015:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776621.596021:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776621.596025:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776621.596030:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776621.596034:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776621.596037:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776621.596039:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776621.660353:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776621.660358:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776621.660361:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776621.660440:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776621.660446:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776621.660448:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776621.660452:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776621.660456:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776621.660460:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776621.660520:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776621.660524:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776621.660527:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776622.619900:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776622.619907:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776622.619908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776622.619914:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776622.619917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776622.619921:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776622.619923:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776622.619926:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776622.619928:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776622.683972:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776622.683979:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776622.683981:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776622.683985:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776622.683988:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776622.683992:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776622.683994:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776622.683996:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776622.683998:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776622.684044:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776622.684049:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776622.684053:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776622.939891:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776622.939902:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776622.939905:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776622.939909:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776622.939913:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776622.939920:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18338 00000400:00000200:0.0:1775776622.939929:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776622.939936:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776622.939939:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776622.939942:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.939945:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.939947:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.939950:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776622.939953:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776622.939955:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776622.939957:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776622.939959:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776622.939960:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776622.939961:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776622.939963:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.939965:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.939966:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.939967:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776622.939969:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776622.939970:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776622.939972:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776622.939974:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776622.939976:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776622.939978:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.939980:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776622.939982:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776622.939989:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776622.939992:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776622.939996:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776622.939997:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940000:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776622.940002:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776622.940006:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776622.940014:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776622.940021:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776622.940030:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099773952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776622.940047:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776622.940052:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776622.940057:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099773952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776622.940077:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776622.940080:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776622.940082:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776622.940083:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776622.940084:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776622.940086:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776622.940087:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18338 00000400:00000200:0.0:1775776622.940091:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776622.940093:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776622.940095:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776622.940097:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940099:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.940101:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940103:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776622.940104:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776622.940106:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776622.940108:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776622.940111:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776622.940114:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776622.940117:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776622.940121:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940123:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.940126:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940133:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776622.940136:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776622.940139:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940143:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776622.940146:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776622.940151:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776622.940156:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940158:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776622.940162:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776622.940167:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940169:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776622.940172:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776622.940175:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776622.940178:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776622.940179:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940183:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776622.940185:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776622.940187:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776622.940191:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776622.940194:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776622.940200:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099774080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776622.940217:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776622.940220:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776622.940227:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099774080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776622.940240:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776622.940242:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776622.940246:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776622.940247:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776622.940248:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776622.940250:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776622.940252:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18338 00000400:00000200:0.0:1775776622.940259:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776622.940264:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776622.940272:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776622.940279:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940283:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.940287:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940290:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776622.940294:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776622.940297:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776622.940300:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776622.940302:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776622.940304:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776622.940308:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776622.940313:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940315:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.940318:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940320:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776622.940324:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776622.940325:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940327:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776622.940328:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776622.940330:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776622.940334:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940338:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776622.940347:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776622.940353:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940356:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776622.940360:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776622.940363:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776622.940364:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776622.940366:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940369:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776622.940370:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776622.940372:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776622.940374:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000010:0.0:1775776622.940376:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000800:00000200:2.1:1775776622.940380:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1775776622.940381:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776622.940386:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099774208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776622.940395:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776622.940397:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776622.940401:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099774208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776622.940409:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776622.940412:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776622.940416:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776622.940418:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776622.940419:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776622.940424:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776622.940430:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18338 00000400:00000200:0.0:1775776622.940436:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776622.940442:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776622.940446:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776622.940450:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940454:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.940459:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940464:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776622.940469:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776622.940470:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776622.940471:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776622.940473:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776622.940477:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776622.940480:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776622.940487:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940492:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.940496:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940499:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776622.940502:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776622.940505:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940507:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776622.940510:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776622.940513:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776622.940517:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940519:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776622.940523:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776622.940525:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940529:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776622.940532:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776622.940533:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776622.940536:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776622.940538:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940539:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776622.940541:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776622.940544:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776622.940548:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776622.940552:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776622.940559:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099774336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776622.940575:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776622.940579:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776622.940584:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099774336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776622.940594:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776622.940598:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776622.940601:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776622.940603:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776622.940606:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776622.940610:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776622.940614:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18338 00000400:00000200:0.0:1775776622.940619:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776622.940624:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776622.940627:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776622.940632:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940636:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.940638:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940640:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776622.940645:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776622.940646:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776622.940649:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776622.940650:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776622.940654:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776622.940656:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776622.940660:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940663:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776622.940683:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940685:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776622.940688:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776622.940690:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776622.940692:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776622.940693:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776622.940696:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776622.940698:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940702:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776622.940705:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776622.940710:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940715:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776622.940719:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776622.940723:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776622.940728:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776622.940731:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776622.940735:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776622.940737:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776622.940738:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776622.940743:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776622.940747:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776622.940754:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099774464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776622.940765:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776622.940767:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776622.940771:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099774464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776622.940801:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776622.940805:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18348) 00000800:00000200:2.1:1775776622.941006:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776622.941008:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776622.941274:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776622.941276:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776622.941535:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776622.941538:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776622.941817:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776622.941819:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1775776623.643909:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776623.643917:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776623.643919:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776623.643924:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776623.643928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776623.643931:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776623.643934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776623.643936:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776623.643939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776623.707873:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776623.707881:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776623.707882:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776623.707887:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776623.707890:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776623.707893:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776623.707894:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776623.707896:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776623.707898:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776623.708056:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776623.708059:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776623.708061:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776624.219875:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313011957, at 4313011968 00000100:00000010:0.0:1775776624.219891:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000040:1.0:1775776624.219958:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313011957, at 4313011968 00000100:00000010:1.0:1775776624.219966:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775776624.667890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776624.667897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776624.667899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776624.667904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776624.667907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776624.667911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776624.667913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776624.667916:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776624.667918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776624.731892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776624.731900:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776624.731903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776624.731909:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776624.731916:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776624.731920:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776624.731923:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776624.731926:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776624.731929:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776624.731978:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776624.731982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776624.731985:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776625.691895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776625.691903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776625.691905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776625.691910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776625.691913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776625.691923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776625.691926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776625.691928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776625.691931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.755890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776625.755897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776625.755900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776625.755914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776625.755919:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776625.755924:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776625.755928:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776625.755930:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776625.755933:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776625.755977:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776625.755980:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776625.755983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776626.715912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776626.715918:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776626.715919:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776626.715923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776626.715925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776626.715928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776626.715930:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776626.715932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776626.715934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776626.779910:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776626.779917:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776626.779920:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776626.779925:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776626.779931:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776626.779936:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776626.779940:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776626.779943:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776626.779947:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776626.780010:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776626.780014:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776626.780017:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776627.739888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776627.739896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776627.739899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776627.739903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776627.739907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776627.739911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776627.739913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776627.739927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776627.739931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776627.804143:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776627.804150:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776627.804152:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776627.804157:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776627.804161:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776627.804165:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776627.804167:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776627.804169:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776627.804172:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776627.804572:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776627.804574:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776627.804577:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776628.059987:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776628.059995:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776628.059997:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776628.060001:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776628.060004:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776628.060009:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18343 00000400:00000200:3.0:1775776628.060017:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776628.060021:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776628.060023:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776628.060026:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060028:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060029:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060031:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776628.060034:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776628.060035:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776628.060037:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776628.060040:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776628.060041:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776628.060043:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776628.060044:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060045:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060046:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060047:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776628.060049:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776628.060051:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060052:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776628.060054:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776628.060057:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776628.060058:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060060:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776628.060061:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776628.060068:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776628.060070:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776628.060073:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776628.060074:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060076:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776628.060078:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776628.060081:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776628.060087:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776628.060092:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776628.060100:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099774592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776628.060110:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776628.060114:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776628.060119:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099774592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776628.060139:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776628.060142:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776628.060146:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776628.060148:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776628.060149:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776628.060151:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776628.060154:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18343 00000400:00000200:3.0:1775776628.060158:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776628.060161:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776628.060164:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776628.060166:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060168:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060170:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060172:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776628.060174:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776628.060177:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776628.060179:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776628.060182:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776628.060186:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776628.060189:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776628.060194:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060199:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060203:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060208:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776628.060211:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776628.060216:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060219:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776628.060222:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776628.060227:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776628.060232:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060235:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776628.060240:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776628.060246:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060249:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776628.060254:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776628.060260:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776628.060264:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776628.060266:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060268:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776628.060270:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776628.060273:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776628.060277:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776628.060283:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776628.060292:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099774720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776628.060301:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776628.060304:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776628.060309:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099774720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776628.060332:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776628.060336:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776628.060342:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776628.060345:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776628.060347:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776628.060352:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776628.060356:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18343 00000400:00000200:3.0:1775776628.060384:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776628.060390:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776628.060394:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776628.060401:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060404:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060408:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060412:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776628.060414:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776628.060416:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776628.060418:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776628.060420:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776628.060424:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776628.060428:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776628.060433:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060437:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775776628.060439:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:3.0:1775776628.060441:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060444:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000800:00000200:2.1:1775776628.060444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000020:00000040:3.0:1775776628.060448:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776628.060452:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060454:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776628.060455:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776628.060458:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776628.060460:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060461:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776628.060463:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776628.060466:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060469:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776628.060473:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776628.060477:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776628.060481:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776628.060482:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060487:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776628.060490:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776628.060492:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776628.060497:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776628.060502:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776628.060510:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099774848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776628.060549:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776628.060553:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776628.060561:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099774848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776628.060593:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776628.060596:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776628.060598:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776628.060600:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776628.060602:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776628.060606:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776628.060609:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18343 00000400:00000200:3.0:1775776628.060617:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776628.060620:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776628.060623:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776628.060628:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060631:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060640:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060643:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776628.060647:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776628.060649:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776628.060651:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776628.060654:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776628.060658:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776628.060664:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776628.060671:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060673:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060677:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060682:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776628.060686:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776628.060692:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060698:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776628.060701:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776628.060705:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776628.060711:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060714:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776628.060718:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776628.060722:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060726:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776628.060732:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776628.060736:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776628.060741:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776628.060744:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060748:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776628.060751:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776628.060754:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776628.060760:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776628.060766:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776628.060774:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099774976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776628.060811:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776628.060814:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776628.060818:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099774976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776628.060827:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776628.060829:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776628.060834:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776628.060837:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776628.060840:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776628.060844:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776628.060848:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18343 00000400:00000200:3.0:1775776628.060855:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776628.060861:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776628.060865:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776628.060871:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060875:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060880:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060884:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776628.060889:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776628.060892:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776628.060894:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776628.060896:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776628.060900:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776628.060903:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776628.060909:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060913:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776628.060915:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060917:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776628.060919:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776628.060924:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776628.060929:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776628.060933:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776628.060938:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776628.060942:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060945:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776628.060948:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776628.060952:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060955:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776628.060961:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776628.060965:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776628.060969:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776628.060972:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.060975:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776628.060978:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776628.060980:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776628.060985:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776628.060991:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776628.060997:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099775104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776628.061010:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776628.061014:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776628.061019:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099775104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776628.061046:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776628.061050:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18353) 00000800:00000200:2.1:1775776628.061134:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776628.061136:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776628.061302:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776628.061304:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776628.061563:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776628.061565:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776628.061948:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776628.061950:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1775776628.763860:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776628.763865:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776628.763866:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776628.763870:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776628.763872:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776628.763875:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776628.763877:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776628.763879:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776628.763880:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776628.827872:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776628.827878:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776628.827879:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776628.827883:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776628.827886:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776628.827889:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776628.827892:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776628.827894:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776628.827897:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776628.829303:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776628.829307:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776628.829311:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776629.339884:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313017077, at 4313017088 00000100:00000040:1.0:1775776629.339884:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313017077, at 4313017088 00000100:00000010:0.0:1775776629.339897:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000010:1.0:1775776629.339902:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:3.0:1775776629.787890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776629.787896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776629.787898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776629.787903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776629.787907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776629.787911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776629.787913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776629.787915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776629.787918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776629.851933:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776629.851939:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776629.851941:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776629.851946:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776629.851949:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776629.851953:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776629.851955:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776629.851957:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776629.851960:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776629.852522:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776629.852525:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776629.852527:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776630.811917:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776630.811926:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776630.811928:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776630.811935:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776630.811939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776630.811944:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776630.811947:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776630.811949:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776630.811953:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776630.875921:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776630.875930:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776630.875934:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776630.875940:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776630.875945:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776630.875951:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776630.875957:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776630.875961:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776630.875966:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776630.876121:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776630.876125:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776630.876128:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776631.835926:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776631.835934:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776631.835936:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776631.835942:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776631.835946:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776631.835951:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776631.835955:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776631.835958:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776631.835960:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.899923:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776631.899932:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776631.899935:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776631.899944:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776631.899950:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776631.899955:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776631.899960:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776631.899964:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776631.899968:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776631.900031:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776631.900036:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776631.900040:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776632.859899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776632.859905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776632.859907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776632.859911:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776632.859913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776632.859916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776632.859918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776632.859919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776632.859921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776632.923907:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776632.923914:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776632.923919:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776632.923928:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776632.923934:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776632.923940:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776632.923944:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776632.923947:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776632.923950:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776632.924011:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776632.924015:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776632.924018:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776633.179953:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776633.179966:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776633.179968:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776633.179972:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776633.179976:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776633.179983:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18348 00000400:00000200:0.0:1775776633.179993:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776633.179999:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776633.180002:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776633.180004:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180007:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180009:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180011:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776633.180015:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776633.180017:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776633.180021:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776633.180024:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776633.180026:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776633.180029:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776633.180032:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180034:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180035:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180037:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776633.180040:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776633.180043:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180046:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776633.180049:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776633.180054:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776633.180056:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180058:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776633.180060:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776633.180071:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776633.180074:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776633.180078:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776633.180080:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180082:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776633.180086:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776633.180090:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776633.180097:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776633.180103:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776633.180113:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099775232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776633.180124:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776633.180129:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776633.180134:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099775232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776633.180160:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776633.180164:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776633.180178:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776633.180179:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776633.180181:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776633.180184:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776633.180187:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18348 00000400:00000200:0.0:1775776633.180192:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776633.180195:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776633.180198:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776633.180201:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180204:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180207:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180209:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776633.180213:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776633.180213:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776633.180215:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776633.180218:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776633.180220:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776633.180223:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776633.180226:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180229:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180233:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180236:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776633.180240:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776633.180251:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180253:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776633.180256:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776633.180262:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776633.180266:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180268:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776633.180274:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776633.180278:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180280:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776633.180285:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776633.180287:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776633.180290:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776633.180292:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180294:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776633.180295:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776633.180296:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776633.180302:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776633.180306:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776633.180312:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099775360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776633.180319:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776633.180321:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776633.180324:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099775360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776633.180332:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776633.180334:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776633.180337:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776633.180339:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776633.180341:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776633.180349:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776633.180353:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18348 00000400:00000200:0.0:1775776633.180360:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776633.180364:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776633.180368:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776633.180372:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180374:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180376:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180379:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776633.180381:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776633.180383:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776633.180386:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776633.180388:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776633.180393:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776633.180394:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776633.180400:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180402:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180407:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180411:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776633.180413:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776633.180417:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180423:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776633.180425:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776633.180429:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776633.180434:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180436:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776633.180439:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776633.180442:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180446:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776633.180452:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776633.180456:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776633.180462:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776633.180465:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180468:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776633.180471:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776633.180475:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776633.180481:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776633.180485:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776633.180503:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099775488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776633.180524:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776633.180527:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776633.180535:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099775488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776633.180550:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776633.180554:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1775776633.180557:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:0.0:1775776633.180559:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776633.180563:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.1:1775776633.180563:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00080000:0.0:1775776633.180565:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776633.180572:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776633.180578:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18348 00000400:00000200:0.0:1775776633.180583:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776633.180589:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776633.180592:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776633.180598:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180601:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180603:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180607:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776633.180612:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776633.180614:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776633.180618:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776633.180621:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776633.180626:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776633.180631:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776633.180638:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180642:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180647:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180651:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776633.180654:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776633.180657:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180662:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776633.180666:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776633.180670:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776633.180681:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180686:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776633.180691:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776633.180694:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180699:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776633.180703:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776633.180708:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776633.180710:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776633.180713:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180716:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776633.180717:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776633.180719:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776633.180723:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776633.180738:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776633.180746:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099775616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776633.180766:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776633.180769:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776633.180773:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099775616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776633.180837:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776633.180842:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776633.180851:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776633.180855:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776633.180857:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776633.180861:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776633.180865:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18348 00000400:00000200:0.0:1775776633.180872:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776633.180879:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776633.180884:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776633.180891:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180895:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180898:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180902:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776633.180908:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776633.180911:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776633.180914:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776633.180918:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776633.180922:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776633.180926:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776633.180933:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180938:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776633.180941:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180946:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776633.180951:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776633.180957:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776633.180961:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776633.180964:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776633.180968:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776633.180972:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180976:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776633.180980:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776633.180985:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.180988:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776633.180994:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776633.180997:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776633.181002:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776633.181005:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776633.181008:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776633.181011:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776633.181013:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776633.181018:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776633.181023:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776633.181030:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099775744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776633.181044:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776633.181046:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776633.181052:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099775744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776633.181068:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776633.181074:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18358) 00000800:00000200:2.1:1775776633.181208:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776633.181211:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776633.181500:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776633.181502:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776633.181801:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776633.181804:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776633.182070:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776633.182073:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1775776633.885061:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776633.885067:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776633.885068:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776633.885073:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776633.885075:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776633.885078:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776633.885080:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776633.885081:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776633.885083:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776633.947890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776633.947897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776633.947900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776633.947904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776633.947908:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776633.947911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776633.947914:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776633.947917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776633.947920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776633.947987:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776633.947991:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776633.947994:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776634.459884:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313022197, at 4313022208 00000100:00000010:1.0:1775776634.459899:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000040:0.0:1775776634.460025:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313022197, at 4313022208 00000100:00000010:0.0:1775776634.460032:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000001:3.0:1775776634.907901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776634.907909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776634.907911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776634.907916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776634.907920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776634.907924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776634.907927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776634.907929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776634.907932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776634.971862:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776634.971869:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776634.971871:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776634.971876:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776634.971880:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776634.971884:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776634.971888:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776634.971892:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776634.971896:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776634.971971:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776634.971975:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776634.971978:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776635.931901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776635.931916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776635.931918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776635.931924:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776635.931928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776635.931932:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776635.931933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776635.931935:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776635.931937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776635.995904:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776635.995913:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776635.995916:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776635.995922:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776635.995928:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776635.995932:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776635.995937:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776635.995940:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776635.995943:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776635.996011:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776635.996015:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776635.996018:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776636.955884:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776636.955890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776636.955891:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776636.955895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776636.955898:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776636.955901:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776636.955903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776636.955904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776636.955906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776637.019890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776637.019897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776637.019899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776637.019904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776637.019908:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776637.019911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776637.019914:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776637.019916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776637.019918:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776637.020048:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776637.020052:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776637.020056:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776637.979891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776637.979896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776637.979898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776637.979901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776637.979904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776637.979907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776637.979909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776637.979911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776637.979912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776638.043887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776638.043893:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776638.043895:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776638.043901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776638.043906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776638.043910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776638.043913:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776638.043917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776638.043921:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776638.044016:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776638.044020:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776638.044023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776638.299879:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776638.299886:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776638.299888:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776638.299891:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776638.299895:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776638.299901:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18353 00000400:00000200:0.0:1775776638.299912:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776638.299917:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776638.299920:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776638.299924:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.299927:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.299929:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.299931:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776638.299935:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776638.299937:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776638.299939:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776638.299942:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776638.299944:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776638.299946:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776638.299948:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.299950:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.299951:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.299953:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776638.299956:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776638.299959:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776638.299961:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776638.299964:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776638.299967:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776638.299970:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.299973:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776638.299975:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776638.299984:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776638.299987:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776638.299991:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776638.299992:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.299994:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776638.299996:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776638.299999:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776638.300006:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776638.300013:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776638.300021:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099775872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776638.300031:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776638.300036:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776638.300041:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099775872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776638.300071:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776638.300074:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776638.300077:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776638.300079:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776638.300080:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776638.300083:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776638.300086:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18353 00000400:00000200:0.0:1775776638.300090:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776638.300093:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776638.300095:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776638.300099:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300101:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.300103:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300105:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776638.300108:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776638.300109:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776638.300111:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776638.300113:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776638.300115:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776638.300118:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776638.300130:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300134:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.300143:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300149:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776638.300155:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776638.300160:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300165:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776638.300169:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776638.300176:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776638.300180:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300185:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776638.300190:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776638.300194:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300198:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776638.300203:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776638.300207:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776638.300212:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776638.300214:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300216:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776638.300217:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776638.300220:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776638.300228:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776638.300233:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776638.300239:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099776000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776638.300246:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776638.300249:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776638.300255:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099776000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776638.300269:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776638.300274:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776638.300281:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776638.300284:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776638.300286:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776638.300290:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776638.300293:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18353 00000400:00000200:0.0:1775776638.300300:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776638.300305:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776638.300309:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776638.300312:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300315:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.300320:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300322:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776638.300326:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776638.300329:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776638.300332:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776638.300335:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776638.300339:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776638.300343:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776638.300350:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300355:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.300360:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300363:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776638.300374:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776638.300379:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300383:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776638.300385:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776638.300389:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776638.300391:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300393:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776638.300395:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776638.300410:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300412:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776638.300415:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776638.300419:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776638.300422:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776638.300423:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300425:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776638.300426:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776638.300429:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776638.300433:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776638.300437:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776638.300441:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00100000:0.0:1775776638.300442:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099776128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775776638.300448:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1775776638.300452:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776638.300454:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776638.300460:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099776128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776638.300470:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776638.300475:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776638.300479:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776638.300480:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776638.300482:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776638.300486:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776638.300490:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18353 00000400:00000200:0.0:1775776638.300497:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776638.300503:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776638.300506:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776638.300512:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300515:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.300518:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300523:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776638.300527:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776638.300529:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776638.300532:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776638.300534:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776638.300537:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776638.300539:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776638.300543:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300547:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.300550:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300554:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776638.300558:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776638.300561:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300565:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776638.300567:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776638.300570:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776638.300571:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300575:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776638.300577:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776638.300580:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300583:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776638.300597:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776638.300601:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776638.300604:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776638.300605:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300609:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776638.300611:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776638.300613:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776638.300618:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776638.300623:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776638.300630:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099776256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776638.300646:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776638.300650:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776638.300658:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099776256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776638.300673:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776638.300677:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776638.300682:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776638.300684:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776638.300696:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776638.300702:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776638.300707:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18353 00000400:00000200:0.0:1775776638.300716:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776638.300722:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776638.300725:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776638.300737:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300740:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.300743:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300746:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776638.300749:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776638.300752:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776638.300755:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776638.300757:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776638.300759:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776638.300761:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776638.300763:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300764:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776638.300765:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300766:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776638.300767:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776638.300769:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776638.300774:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776638.300798:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776638.300802:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776638.300804:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300806:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776638.300808:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776638.300813:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300817:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776638.300821:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776638.300826:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776638.300830:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776638.300832:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776638.300835:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776638.300837:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776638.300839:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776638.300842:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776638.300847:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776638.300854:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099776384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776638.300866:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776638.300869:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776638.300875:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099776384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776638.300888:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776638.300895:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18363) 00000800:00000200:2.1:1775776638.301067:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776638.301069:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776638.301339:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776638.301341:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776638.301609:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776638.301611:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776638.301861:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776638.301862:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1775776639.003894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776639.003902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776639.003904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776639.003909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776639.003912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776639.003916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776639.003918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776639.003920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776639.003922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776639.067893:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776639.067901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776639.067903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776639.067907:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776639.067911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776639.067915:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776639.067917:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776639.067920:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776639.067922:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776639.068308:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776639.068312:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776639.068315:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776639.579877:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313027317, at 4313027328 00000100:00000040:1.0:1775776639.579877:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313027317, at 4313027328 00000100:00000010:0.0:1775776639.579891:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000010:1.0:1775776639.579891:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000001:3.0:1775776640.027904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776640.027911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776640.027912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776640.027916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776640.027920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776640.027923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776640.027925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776640.027927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776640.027929:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776640.092025:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776640.092029:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776640.092032:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776640.093438:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776640.093442:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776640.093444:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776640.093448:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776640.093451:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776640.093454:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776640.093456:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776640.093458:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776640.093460:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776641.052117:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776641.052195:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776641.052197:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776641.052202:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776641.052205:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776641.052209:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776641.052212:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776641.052214:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776641.052216:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776641.115887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776641.115896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776641.115898:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776641.115902:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776641.115905:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776641.115908:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776641.115910:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776641.115912:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776641.115914:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776641.115976:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776641.115980:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776641.115983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776642.075912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776642.075929:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776642.075931:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776642.075936:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776642.075940:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776642.075943:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776642.075946:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776642.075948:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776642.075951:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776642.139890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776642.139897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776642.139898:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776642.139904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776642.139907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776642.139910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776642.139912:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776642.139915:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776642.139918:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776642.140453:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776642.140457:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776642.140461:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776643.099897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776643.099904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776643.099905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776643.099911:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776643.099914:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776643.099917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776643.099920:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776643.099922:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776643.099924:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.163891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776643.163896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776643.163899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776643.163905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776643.163909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776643.163912:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776643.163915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776643.163918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776643.163921:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776643.163978:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776643.163982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776643.163986:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776643.420378:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776643.420388:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776643.420391:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776643.420394:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776643.420398:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776643.420406:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18358 00000400:00000200:3.0:1775776643.420416:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776643.420423:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776643.420426:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776643.420430:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420433:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.420435:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420438:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776643.420441:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776643.420444:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776643.420447:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776643.420449:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776643.420452:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776643.420454:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776643.420457:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420459:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.420461:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420463:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776643.420466:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776643.420469:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420471:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776643.420474:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776643.420477:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776643.420479:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420482:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776643.420484:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776643.420492:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776643.420495:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776643.420499:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776643.420500:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420503:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776643.420506:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776643.420509:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776643.420516:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776643.420522:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776643.420531:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776643.420541:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776643.420545:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776643.420549:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776643.420582:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776643.420586:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776643.420589:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776643.420590:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776643.420591:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776643.420594:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776643.420596:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18358 00000400:00000200:3.0:1775776643.420600:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776643.420603:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776643.420605:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776643.420607:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420609:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.420611:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420613:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776643.420615:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776643.420616:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776643.420617:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776643.420619:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776643.420621:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776643.420623:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776643.420625:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420627:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.420629:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420631:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776643.420633:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776643.420635:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420637:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776643.420639:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776643.420644:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776643.420646:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420648:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776643.420652:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776643.420654:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420656:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776643.420659:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775776643.420661:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776643.420664:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776643.420665:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420666:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776643.420668:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776643.420669:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776643.420673:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775776643.420676:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776643.420680:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776643.420688:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776643.420690:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776643.420693:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776643.420701:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776643.420703:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776643.420706:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776643.420707:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776643.420709:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776643.420711:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776643.420713:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18358 00000400:00000200:3.0:1775776643.420717:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776643.420720:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776643.420721:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776643.420724:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420726:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.420728:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420730:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776643.420731:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776643.420733:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776643.420734:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776643.420736:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776643.420738:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776643.420739:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776643.420742:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420744:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.420745:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420747:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776643.420749:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776643.420752:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420754:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776643.420756:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776643.420758:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776643.420760:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420762:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776643.420763:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776643.420766:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420767:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776643.420772:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776643.420774:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776643.420796:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776643.420798:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420800:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776643.420801:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776643.420803:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776643.420806:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776643.420810:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776643.420815:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776643.420821:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776643.420823:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776643.420826:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776643.420833:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776643.420835:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776643.420838:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776643.420840:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776643.420841:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776643.420843:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776643.420846:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18358 00000400:00000200:3.0:1775776643.420849:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776643.420852:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776643.420854:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776643.420857:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420859:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.420860:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420862:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776643.420864:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776643.420866:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776643.420867:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776643.420869:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776643.420870:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776643.420872:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776643.420874:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420876:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.420878:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420879:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776643.420881:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776643.420883:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776643.420885:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776643.420886:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776643.420889:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776643.420891:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420893:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776643.420895:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776643.420897:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420898:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776643.420901:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776643.420903:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776643.420906:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776643.420907:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.420908:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776643.420910:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776643.420913:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776643.420917:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776643.420921:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776643.420926:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776643.420936:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776643.420940:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776643.420947:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776643.420961:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776643.420965:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776643.420972:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776643.420974:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776643.420976:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776643.420980:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776643.420984:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18358 00000400:00000200:3.0:1775776643.420989:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776643.420993:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776643.420997:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776643.421001:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.421004:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.421007:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.421010:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776643.421012:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776643.421014:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776643.421017:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776643.421019:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776643.421024:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776643.421029:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776643.421036:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.421040:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776643.421044:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776643.421048:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776643.421051:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776643.421057:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776643.421062:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776643.421063:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776643.421067:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776643.421070:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.421074:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776643.421077:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776643.421080:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.421082:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776643.421088:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776643.421091:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776643.421094:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776643.421097:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776643.421100:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776643.421102:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776643.421105:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776643.421111:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776643.421117:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776643.421126:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776643.421142:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776643.421146:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776643.421152:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776643.421167:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776643.421168:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:3.0:1775776643.421172:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18368) 00000800:00000200:2.1:1775776643.421337:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776643.421452:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776643.421587:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776643.421700:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776644.123905:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776644.123914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776644.123916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776644.123921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776644.123925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776644.123928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776644.123932:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776644.123934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776644.123937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776644.187897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776644.187905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776644.187908:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776644.187915:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776644.187920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776644.187925:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776644.187929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776644.187933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776644.187937:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776644.188001:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776644.188005:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776644.188008:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776644.699900:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313032438, at 4313032448 00000100:00000010:1.0:1775776644.699914:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000040:0.0:1775776644.700176:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313032438, at 4313032448 00000100:00000010:0.0:1775776644.700183:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:3.0:1775776645.147946:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776645.147952:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776645.147953:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776645.147959:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776645.147963:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776645.147966:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776645.147969:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776645.147971:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776645.147974:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776645.211890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776645.211903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776645.211907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776645.211914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776645.211918:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776645.211922:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776645.211925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776645.211927:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776645.211931:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776645.212118:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776645.212122:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776645.212125:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776646.171895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776646.171903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776646.171905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776646.171910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776646.171912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776646.171915:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.171917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776646.171919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.171921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776646.235897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776646.235905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776646.235908:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776646.235914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776646.235921:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776646.235926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.235929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776646.235933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.235937:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776646.236012:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776646.236016:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776646.236019:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776646.494094:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776646.494100:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776646.494104:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776646.494111:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776643/real 1775776646] req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776646.494125:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776646.494127:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776646.494130:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601c528. 00000400:00000200:3.0:1775776646.494133:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dbb0 00000400:00000010:3.0:1775776646.494135:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dbb0. 00000100:00000001:3.0:1775776646.494138:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776646.494142:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776646.494149:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494155:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776646.494157:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494160:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494162:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776646.494165:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494172:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776646.494177:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494184:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776646.494187:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494188:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494189:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494192:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776646.494194:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776646.494198:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776646.494202:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776646.494205:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776646.494209:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776646.494216:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099776896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776646.494220:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776646.494224:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776643/real 1775776646] req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776646.494231:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776646.494232:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776646.494234:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601c840. 00000400:00000200:3.0:1775776646.494236:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d5d8 00000400:00000010:3.0:1775776646.494237:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d5d8. 00000100:00000001:3.0:1775776646.494239:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776646.494242:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776646.494249:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494254:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776646.494270:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494271:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494272:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776646.494276:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494282:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776646.494285:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494291:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776646.494294:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494296:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494297:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494298:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776646.494300:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776646.494303:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776646.494307:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776646.494309:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776646.494312:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776646.494321:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099777024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776646.494325:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776646.494328:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776643/real 1775776646] req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776646.494337:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776646.494338:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776646.494340:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601c1b8. 00000400:00000200:3.0:1775776646.494342:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720df68 00000400:00000010:3.0:1775776646.494344:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720df68. 00000100:00000001:3.0:1775776646.494347:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776646.494350:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776646.494357:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494363:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776646.494364:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494366:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494367:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776646.494371:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494378:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776646.494381:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494387:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776646.494389:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494391:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494392:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494394:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776646.494395:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776646.494397:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776646.494400:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776646.494403:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776646.494406:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776646.494413:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099776512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776646.494418:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776646.494422:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776643/real 1775776646] req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776646.494428:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776646.494430:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776646.494432:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cdc0. 00000400:00000200:3.0:1775776646.494434:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d6e8 00000400:00000010:3.0:1775776646.494435:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d6e8. 00000100:00000001:3.0:1775776646.494437:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776646.494440:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776646.494449:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494456:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776646.494457:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494459:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494460:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776646.494463:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494469:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776646.494473:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494479:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776646.494481:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494482:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494483:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494485:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776646.494487:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776646.494489:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776646.494493:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776646.494495:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776646.494498:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776646.494505:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099776640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776646.494509:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776646.494512:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776643/real 1775776646] req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776646.494520:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776646.494521:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776646.494523:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cf20. 00000400:00000200:3.0:1775776646.494525:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720da18 00000400:00000010:3.0:1775776646.494526:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720da18. 00000100:00000001:3.0:1775776646.494528:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776646.494530:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776646.494537:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494543:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776646.494544:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494545:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494546:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776646.494550:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494556:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776646.494560:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494566:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776646.494567:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494569:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776646.494570:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494571:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776646.494572:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776646.494574:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776646.494577:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776646.494579:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776646.494583:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776646.494590:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099776768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776646.494594:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494596:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776646.494598:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099776896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494605:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776646.494608:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776646.494613:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506000. 02000000:00000001:3.0:1775776646.494616:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776646.494630:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776646.494632:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776646.494635:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776646.494638:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775776646.494642:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776646.494644:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776646.494646:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:3.0:1775776646.494650:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776646.494651:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494653:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776646.494656:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099777024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494663:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776646.494664:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776646.494666:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:3.0:1775776646.494669:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776646.494670:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776646.494671:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776646.494673:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776646.494675:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:3.0:1775776646.494677:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776646.494678:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776646.494679:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:3.0:1775776646.494682:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776646.494683:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494684:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776646.494687:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099776512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494694:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776646.494696:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776646.494697:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:3.0:1775776646.494700:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776646.494701:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776646.494702:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776646.494704:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776646.494705:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776646.494707:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776646.494708:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776646.494710:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:3.0:1775776646.494712:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776646.494713:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494715:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776646.494718:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def800 x1862033099776640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494725:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776646.494726:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776646.494727:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505c00. 02000000:00000001:3.0:1775776646.494729:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776646.494730:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776646.494732:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776646.494734:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776646.494735:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776646.494737:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776646.494738:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776646.494739:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def800. 00000100:00000001:3.0:1775776646.494741:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776646.494742:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494744:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776646.494747:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedf80 x1862033099776768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776668 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776646.494753:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776646.494754:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776646.494756:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:3.0:1775776646.494757:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776646.494759:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776646.494760:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776646.494761:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776646.494763:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776646.494764:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776646.494765:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776646.494766:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedf80. 00000100:00000001:3.0:1775776646.494767:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776646.494768:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494771:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776646.494814:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776646.494816:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776646.494818:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776646.494819:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776647.195958:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776647.195966:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776647.195968:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776647.195973:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776647.195976:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776647.195980:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776647.195982:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776647.195984:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776647.195987:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776647.259909:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776647.259916:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776647.259918:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776647.259922:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776647.259926:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776647.259928:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776647.259931:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776647.259933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776647.259936:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776647.260035:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776647.260039:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776647.260042:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776648.219897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776648.219903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776648.219905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776648.219909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776648.219913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776648.219916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776648.219919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776648.219921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776648.219923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.283897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776648.283903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776648.283905:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776648.283910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776648.283913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776648.283917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776648.283919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776648.283922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776648.283924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776648.284996:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776648.285000:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776648.285011:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776648.539914:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776648.539924:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776648.539927:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776648.539932:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776648.539936:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776648.539944:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18363 00000400:00000200:3.0:1775776648.539953:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776648.539959:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776648.539962:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776648.539966:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.539969:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.539971:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.539974:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776648.539977:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776648.539979:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776648.539982:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776648.539985:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776648.539987:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776648.539989:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776648.539993:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.539995:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.539997:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.539999:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776648.540002:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776648.540005:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540008:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776648.540010:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776648.540013:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776648.540016:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540018:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776648.540021:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776648.540027:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776648.540031:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776648.540035:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776648.540036:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540039:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776648.540041:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776648.540045:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776648.540052:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776648.540058:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776648.540066:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099777152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776648.540075:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776648.540079:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776648.540084:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099777152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776648.540108:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776648.540110:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776648.540114:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776648.540115:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776648.540117:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776648.540120:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776648.540122:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18363 00000400:00000200:3.0:1775776648.540125:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776648.540128:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776648.540130:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776648.540133:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540135:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.540138:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540140:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776648.540142:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776648.540143:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776648.540145:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776648.540147:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776648.540149:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776648.540151:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776648.540155:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540158:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.540162:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540166:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776648.540168:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776648.540175:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540180:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776648.540183:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776648.540190:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776648.540195:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540199:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776648.540205:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776648.540211:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540216:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776648.540239:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775776648.540242:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776648.540244:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776648.540245:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540247:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776648.540248:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776648.540251:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776648.540257:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775776648.540261:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776648.540268:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099777280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776648.540278:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776648.540283:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776648.540288:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099777280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776648.540309:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776648.540313:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776648.540319:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776648.540322:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776648.540325:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776648.540329:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776648.540335:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18363 00000400:00000200:3.0:1775776648.540343:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776648.540349:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776648.540354:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776648.540359:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540363:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.540369:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540373:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776648.540378:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776648.540381:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776648.540383:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776648.540385:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776648.540387:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776648.540391:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776648.540396:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540399:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.540408:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540413:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776648.540418:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776648.540423:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540429:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776648.540431:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776648.540435:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776648.540439:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540443:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776648.540446:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776648.540449:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540453:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776648.540458:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776648.540462:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776648.540468:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776648.540470:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540475:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776648.540477:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776648.540478:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776648.540482:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776648.540487:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776648.540497:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099777408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776648.540508:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776648.540511:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776648.540518:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099777408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776648.540532:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776648.540536:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:3.0:1775776648.540537:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1775776648.540541:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00080000:3.0:1775776648.540542:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776648.540545:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776648.540547:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776648.540553:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776648.540558:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18363 00000400:00000200:3.0:1775776648.540563:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776648.540570:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776648.540574:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776648.540579:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540582:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.540585:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540589:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776648.540591:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776648.540594:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776648.540596:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776648.540598:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776648.540601:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776648.540606:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776648.540613:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540617:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.540620:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540623:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776648.540627:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776648.540634:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776648.540639:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776648.540644:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776648.540648:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776648.540652:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.540655:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.1:1775776648.541015:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776648.541017:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776648.541306:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776648.541308:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000008:00100000:3.0:1775776648.544363:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776648.544371:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.544374:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776648.544384:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776648.544388:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776648.544393:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776648.544395:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.544398:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776648.544400:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776648.544405:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776648.544412:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776648.544418:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776648.544429:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099777536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776648.544444:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776648.544449:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776648.544453:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099777536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776648.544480:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776648.544485:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776648.544489:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776648.544491:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776648.544493:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776648.544496:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776648.544499:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18363 00000400:00000200:3.0:1775776648.544508:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776648.544514:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776648.544518:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776648.544522:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.544525:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.544528:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.544531:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776648.544534:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776648.544535:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776648.544538:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776648.544543:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776648.544547:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776648.544551:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776648.544557:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.544561:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776648.544565:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776648.544569:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776648.544574:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776648.544579:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776648.544584:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776648.544587:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776648.544595:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776648.544600:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.544604:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776648.544608:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776648.544612:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.544615:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776648.544621:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776648.544624:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776648.544630:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776648.544632:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776648.544635:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776648.544639:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776648.544641:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776648.544647:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776648.544652:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776648.544660:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099777664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776648.544674:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776648.544677:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776648.544684:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099777664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776648.544701:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776648.544708:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18373) 00000800:00000200:2.1:1775776648.544828:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776648.544831:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776648.545172:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776648.545174:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1775776649.243904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776649.243911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776649.243913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776649.243919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776649.243922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776649.243927:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776649.243929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776649.243931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776649.243933:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776649.307895:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776649.307905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776649.307908:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776649.307914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776649.307920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776649.307925:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776649.307929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776649.307933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776649.307937:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776649.307996:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776649.307999:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776649.308002:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776649.819883:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313037557, at 4313037568 00000100:00000040:1.0:1775776649.819894:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313037557, at 4313037568 00000100:00000010:0.0:1775776649.819899:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000010:1.0:1775776649.819902:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775776650.267944:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776650.267950:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776650.267952:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776650.267958:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776650.267961:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776650.267965:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776650.267967:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776650.267970:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776650.267972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776650.332022:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776650.332025:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776650.332028:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776650.333263:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776650.333267:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776650.333269:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776650.333274:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776650.333277:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776650.333281:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776650.333283:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776650.333286:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776650.333288:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776651.292082:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776651.292089:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776651.292091:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776651.292097:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776651.292100:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776651.292104:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776651.292107:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776651.292109:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776651.292112:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776651.355906:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776651.355914:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776651.355917:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776651.355922:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776651.355927:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776651.355932:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776651.355936:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776651.355939:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776651.355942:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776651.355997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776651.356000:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776651.356003:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776652.315896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776652.315902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776652.315904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776652.315908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776652.315910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776652.315913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776652.315915:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776652.315917:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776652.315919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776652.379966:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776652.379970:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776652.379973:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776652.380202:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776652.380207:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776652.380209:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776652.380213:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776652.380217:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776652.380220:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776652.380223:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776652.380225:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776652.380228:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.339895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776653.339902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776653.339904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776653.339909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776653.339912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776653.339916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776653.339919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776653.339921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776653.339924:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.403891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776653.403899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776653.403901:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776653.403908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776653.403913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776653.403917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776653.403921:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776653.403925:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776653.403929:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776653.404004:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776653.404007:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776653.404010:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776653.659851:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776653.659861:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776653.659864:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776653.659868:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776653.659872:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776653.659879:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18368 00000400:00000200:3.0:1775776653.659889:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776653.659896:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776653.659899:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776653.659903:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.659906:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.659908:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.659911:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776653.659914:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776653.659916:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776653.659919:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776653.659922:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776653.659924:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776653.659926:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776653.659929:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.659930:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.659932:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.659934:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776653.659937:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776653.659940:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776653.659943:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776653.659946:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776653.659949:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776653.659952:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.659955:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776653.659957:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776653.659965:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776653.659969:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776653.659972:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776653.659974:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.659977:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776653.659979:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776653.659983:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776653.659990:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776653.659996:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776653.660003:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099777792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776653.660019:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776653.660022:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776653.660027:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099777792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776653.660055:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776653.660058:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776653.660061:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776653.660062:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776653.660064:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776653.660066:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776653.660069:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18368 00000400:00000200:3.0:1775776653.660073:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776653.660077:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776653.660079:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776653.660082:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660085:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.660089:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660093:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776653.660097:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776653.660100:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776653.660103:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776653.660106:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776653.660111:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776653.660115:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776653.660121:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660126:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.660145:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660149:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776653.660154:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776653.660157:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660162:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776653.660165:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776653.660171:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776653.660174:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660176:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776653.660181:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776653.660186:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660190:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776653.660196:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776653.660200:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776653.660205:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776653.660209:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660212:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776653.660215:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776653.660217:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776653.660223:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776653.660227:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776653.660236:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099777920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776653.660247:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776653.660259:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776653.660266:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099777920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776653.660280:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776653.660284:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776653.660291:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776653.660294:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776653.660296:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776653.660300:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776653.660305:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18368 00000400:00000200:3.0:1775776653.660311:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776653.660317:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776653.660321:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776653.660327:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776653.660330:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776653.660332:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775776653.660335:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000100:00000040:3.0:1775776653.660337:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660343:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776653.660346:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776653.660349:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776653.660352:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776653.660354:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776653.660356:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776653.660360:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776653.660366:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660369:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.660380:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660383:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776653.660388:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776653.660395:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660398:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776653.660402:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776653.660406:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776653.660411:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660416:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776653.660420:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776653.660424:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660427:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776653.660431:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776653.660435:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776653.660441:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776653.660444:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660447:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776653.660449:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776653.660451:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776653.660457:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776653.660461:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776653.660468:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099778048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776653.660482:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776653.660486:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776653.660493:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099778048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776653.660510:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776653.660513:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776653.660518:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776653.660522:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776653.660524:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776653.660527:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776653.660532:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18368 00000400:00000200:3.0:1775776653.660541:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776653.660549:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776653.660552:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776653.660559:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660565:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.660569:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660574:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776653.660578:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776653.660580:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776653.660584:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776653.660587:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776653.660591:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776653.660596:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776653.660604:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660609:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.660614:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660619:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776653.660624:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776653.660631:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660636:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776653.660640:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776653.660644:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776653.660649:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660653:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776653.660656:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776653.660661:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660665:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776653.660671:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776653.660675:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776653.660678:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776653.660679:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660682:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776653.660685:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776653.660687:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776653.660693:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776653.660698:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776653.660706:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099778176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776653.660718:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776653.660722:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776653.660729:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099778176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776653.660746:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776653.660749:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776653.660752:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776653.660753:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776653.660754:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776653.660757:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776653.660760:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18368 00000400:00000200:3.0:1775776653.660763:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776653.660767:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776653.660769:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776653.660772:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660774:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.660794:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660797:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776653.660799:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776653.660800:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776653.660801:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776653.660803:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776653.660804:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776653.660806:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776653.660809:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660811:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776653.660812:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660814:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776653.660816:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776653.660818:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776653.660820:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776653.660822:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776653.660825:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776653.660826:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660828:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776653.660830:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776653.660832:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660834:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776653.660837:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776653.660839:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776653.660841:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776653.660842:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776653.660844:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776653.660845:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776653.660846:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776653.660849:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776653.660852:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776653.660856:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099778304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776653.660862:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776653.660864:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776653.660867:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099778304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776653.660873:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776653.660877:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18379) 00000800:00000200:2.1:1775776653.661078:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776653.661080:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775776653.661347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776653.661349:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775776653.661578:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776653.661580:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1775776653.661813:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776653.661815:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 52 secs later) 00000100:00000001:3.0:1775776654.363873:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776654.363879:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776654.363881:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776654.363886:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776654.363889:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776654.363893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776654.363895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776654.363898:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776654.363900:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776654.427906:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776654.427913:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776654.427916:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776654.427920:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776654.427925:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776654.427929:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776654.427932:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776654.427934:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776654.427937:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776654.427987:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776654.427990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776654.427992:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776654.939887:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313042677, at 4313042688 00000100:00000040:0.0:1775776654.939888:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313042677, at 4313042688 00000100:00000010:0.0:1775776654.939901:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000010:1.0:1775776654.939903:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000001:3.0:1775776655.388225:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776655.388233:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776655.388235:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776655.388240:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776655.388244:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776655.388248:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776655.388250:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776655.388253:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776655.388255:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776655.451888:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776655.451895:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776655.451896:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776655.451899:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776655.451903:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776655.451905:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776655.451908:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776655.451910:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776655.451912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776655.452001:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776655.452005:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776655.452008:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776656.411912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776656.411919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776656.411921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776656.411925:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776656.411928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776656.411931:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776656.411934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776656.411936:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776656.411939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776656.475886:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776656.475893:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776656.475896:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776656.475902:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776656.475906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776656.475910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776656.475913:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776656.475916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776656.475919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776656.476058:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776656.476060:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776656.476062:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776657.435867:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776657.435886:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776657.435888:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776657.435893:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776657.435897:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776657.435900:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776657.435903:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776657.435905:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776657.435908:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776657.500142:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776657.500147:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776657.500149:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776657.500152:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00100000:3.0:1775776657.500153:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000020:00000001:1.0:1775776657.500156:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776657.500158:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00010000:1.0:1775776657.500159:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776657.500163:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776657.500167:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776657.500170:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776657.500173:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.459903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776658.459910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776658.459912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776658.459917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776658.459920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776658.459924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776658.459926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776658.459928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776658.459931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.524124:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776658.524131:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776658.524134:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776658.524140:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776658.524145:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776658.524150:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776658.524154:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776658.524157:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776658.524161:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776658.524317:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776658.524321:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776658.524324:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776658.779867:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776658.779877:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776658.779879:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776658.779883:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776658.779887:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776658.779894:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18374 00000400:00000200:3.0:1775776658.779903:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776658.779908:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776658.779911:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776658.779913:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.779915:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.779917:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.779918:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776658.779921:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776658.779922:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776658.779924:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776658.779926:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776658.779927:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776658.779929:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776658.779931:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.779932:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.779933:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.779934:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776658.779935:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776658.779937:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776658.779938:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776658.779940:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776658.779942:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776658.779944:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.779945:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776658.779947:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776658.779954:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776658.779957:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776658.779966:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776658.779967:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.779969:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776658.779970:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776658.779974:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776658.779979:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776658.779985:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776658.779991:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099778432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776658.779997:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776658.780001:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776658.780003:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099778432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776658.780019:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776658.780022:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776658.780028:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776658.780029:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776658.780031:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776658.780033:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776658.780035:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18374 00000400:00000200:3.0:1775776658.780038:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776658.780056:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776658.780058:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776658.780060:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780062:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.780064:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780066:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776658.780068:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776658.780069:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776658.780071:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776658.780073:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776658.780074:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776658.780076:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776658.780078:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780080:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.780082:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780083:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776658.780085:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776658.780088:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780090:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776658.780092:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776658.780096:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776658.780098:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780100:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776658.780104:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776658.780106:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780108:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776658.780111:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776658.780113:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776658.780115:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776658.780117:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780118:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776658.780119:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776658.780121:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776658.780124:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775776658.780127:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776658.780131:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099778560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776658.780138:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776658.780140:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776658.780143:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099778560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776658.780155:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776658.780158:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776658.780161:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776658.780162:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776658.780163:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776658.780166:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776658.780168:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18374 00000400:00000200:3.0:1775776658.780171:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776658.780174:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776658.780176:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776658.780178:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780180:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.780182:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780183:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776658.780185:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776658.780186:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776658.780188:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776658.780189:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776658.780190:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776658.780193:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776658.780195:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780197:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.780198:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780200:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776658.780202:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776658.780204:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780206:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776658.780208:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776658.780210:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776658.780212:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780214:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776658.780220:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776658.780222:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780223:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776658.780227:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776658.780229:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776658.780231:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776658.780232:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780234:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776658.780235:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776658.780237:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776658.780240:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776658.780242:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776658.780246:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099778688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776658.780252:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1775776658.780252:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776658.780254:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776658.780255:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000100:00000040:3.0:1775776658.780257:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099778688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776658.780264:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776658.780266:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776658.780269:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776658.780270:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776658.780271:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776658.780274:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776658.780275:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18374 00000400:00000200:3.0:1775776658.780278:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776658.780287:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776658.780290:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776658.780292:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780294:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.780296:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780297:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776658.780300:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776658.780301:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776658.780303:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776658.780304:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776658.780306:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776658.780307:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776658.780309:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780311:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.780312:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780314:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776658.780316:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776658.780318:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780320:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776658.780322:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776658.780325:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776658.780327:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780328:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776658.780330:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776658.780332:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780333:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776658.780336:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776658.780339:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776658.780341:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776658.780342:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780344:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776658.780345:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776658.780346:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776658.780349:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776658.780353:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776658.780357:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099778816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776658.780364:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776658.780366:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776658.780370:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099778816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776658.780377:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776658.780379:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776658.780382:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776658.780383:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776658.780385:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776658.780387:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776658.780389:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18374 00000400:00000200:3.0:1775776658.780393:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776658.780395:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776658.780403:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776658.780405:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780407:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.780409:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780411:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776658.780413:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776658.780414:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776658.780416:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776658.780417:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776658.780419:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776658.780421:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776658.780423:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780425:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776658.780427:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780429:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776658.780431:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776658.780433:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776658.780435:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776658.780437:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776658.780439:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776658.780441:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780442:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776658.780444:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776658.780446:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780448:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776658.780451:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776658.780452:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776658.780455:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776658.780456:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776658.780457:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776658.780458:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776658.780460:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776658.780464:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776658.780467:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776658.780470:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099778944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776658.780482:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776658.780484:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776658.780486:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099778944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776658.780493:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776658.780496:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18384) 00000800:00000200:2.1:1775776658.780606:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776658.780607:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776658.780888:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776658.780890:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776658.781068:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776658.781069:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776658.781221:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776658.781222:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1775776659.483869:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776659.483876:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776659.483878:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776659.483883:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776659.483887:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776659.483891:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776659.483893:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776659.483895:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776659.483898:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776659.547911:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776659.547918:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776659.547921:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776659.547927:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776659.547933:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776659.547936:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776659.547948:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776659.547951:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776659.547955:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776659.548005:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776659.548008:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776659.548011:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776660.059898:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313047797, at 4313047808 00000100:00000010:1.0:1775776660.059923:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000040:0.0:1775776660.060163:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313047797, at 4313047808 00000100:00000010:0.0:1775776660.060171:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000001:3.0:1775776660.507892:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776660.507897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776660.507898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776660.507902:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776660.507905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776660.507908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776660.507910:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776660.507911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776660.507913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776660.571884:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776660.571890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776660.571893:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776660.571897:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776660.571900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776660.571904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776660.571907:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776660.571909:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776660.571912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776660.571992:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776660.571996:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776660.571999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776661.531934:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776661.531941:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776661.531943:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776661.531948:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776661.531951:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776661.531956:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776661.531959:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776661.531961:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776661.531963:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776661.596039:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776661.596042:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776661.596045:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776661.596704:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776661.596709:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776661.596710:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776661.596714:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776661.596717:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776661.596720:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776661.596723:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776661.596725:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776661.596728:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776662.555907:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776662.555914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776662.555916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776662.555921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776662.555925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776662.555928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776662.555931:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776662.555934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776662.555936:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776662.619998:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776662.620002:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776662.620005:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776662.620617:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776662.620621:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776662.620623:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776662.620626:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776662.620630:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776662.620633:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776662.620635:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776662.620638:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776662.620640:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776663.579889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776663.579896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776663.579898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776663.579904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776663.579908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776663.579911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776663.579914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776663.579917:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776663.579919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776663.643909:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776663.643917:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776663.643919:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776663.643925:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776663.643928:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776663.643932:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776663.643935:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776663.643938:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776663.643941:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776663.644334:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776663.644339:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776663.644342:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776663.899930:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776663.899940:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776663.899943:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776663.899947:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776663.899977:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776663.899985:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18379 00000400:00000200:0.0:1775776663.899996:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776663.900003:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776663.900005:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776663.900008:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900012:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.900014:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900016:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776663.900020:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776663.900022:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776663.900025:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776663.900028:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776663.900030:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776663.900032:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776663.900035:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900037:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.900039:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900041:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776663.900043:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776663.900046:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900049:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776663.900051:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776663.900054:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776663.900057:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.900059:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776663.900061:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776663.900071:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776663.900074:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776663.900078:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776663.900080:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.900083:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776663.900085:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776663.900089:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776663.900097:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776663.900104:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776663.900113:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099779072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776663.900124:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776663.900128:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776663.900133:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099779072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776663.900156:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776663.900159:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776663.900162:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776663.900164:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776663.900165:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776663.900168:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776663.900170:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18379 00000400:00000200:0.0:1775776663.900174:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776663.900177:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776663.900179:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776663.900183:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900185:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.900188:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900190:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776663.900193:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776663.900196:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776663.900198:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776663.900201:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776663.900204:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776663.900207:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776663.900210:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900213:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.900219:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900223:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776663.900231:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776663.900239:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900247:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776663.900249:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776663.900254:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776663.900267:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.900280:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776663.900287:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776663.900292:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.900297:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776663.900304:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776663.900313:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776663.900321:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776663.900327:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.900331:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776663.900333:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776663.900337:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776663.900349:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776663.900354:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776663.900369:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099779200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776663.900384:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776663.900390:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776663.900412:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099779200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776663.900440:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776663.900448:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776663.900452:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776663.900454:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776663.900458:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776663.900467:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776663.900480:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18379 00000400:00000200:0.0:1775776663.900497:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776663.900516:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776663.900520:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776663.900536:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900546:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.900553:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900557:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776663.900565:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776663.900569:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776663.900574:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.1:1775776663.900575:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:0.0:1775776663.900579:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000800:00000200:2.1:1775776663.900580:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000020:00000001:0.0:1775776663.900594:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776663.900605:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776663.900614:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900623:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.900630:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900638:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776663.900660:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776663.900675:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776663.900679:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776663.900690:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776663.900698:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776663.900703:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.900708:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776663.900721:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776663.900729:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.900736:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776663.900749:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776663.900767:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776663.900776:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776663.900844:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.900856:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776663.900861:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776663.900868:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776663.900878:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776663.900883:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776663.900893:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099779328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776663.900918:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776663.900924:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776663.900935:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099779328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776663.900997:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776663.901006:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776663.901014:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776663.901019:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776663.901025:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776663.901031:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776663.901038:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18379 00000400:00000200:0.0:1775776663.901051:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776663.901057:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776663.901060:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776663.901072:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901076:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.901084:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901090:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776663.901097:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776663.901100:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776663.901103:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776663.901115:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776663.901125:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776663.901128:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776663.901135:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901140:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.901144:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901149:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776663.901154:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776663.901162:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901169:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776663.901177:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776663.901182:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776663.901188:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.901194:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776663.901198:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776663.901207:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.901214:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776663.901225:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776663.901231:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776663.901237:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776663.901242:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.901246:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776663.901249:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776663.901253:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776663.901264:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776663.901269:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776663.901275:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099779456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776663.901282:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776663.901283:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776663.901287:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099779456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776663.901295:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776663.901298:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776663.901300:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776663.901302:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776663.901303:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776663.901305:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776663.901308:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18379 00000400:00000200:0.0:1775776663.901312:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776663.901314:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776663.901316:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776663.901318:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901320:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.901322:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901323:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776663.901325:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776663.901326:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776663.901327:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776663.901328:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776663.901330:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776663.901331:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776663.901334:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901335:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776663.901337:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901338:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776663.901340:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776663.901342:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776663.901343:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776663.901345:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776663.901348:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776663.901349:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.901351:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776663.901352:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776663.901354:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.901356:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776663.901359:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776663.901360:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776663.901363:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776663.901364:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776663.901366:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776663.901367:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776663.901368:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776663.901371:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776663.901374:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776663.901378:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099779584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776663.901383:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776663.901385:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776663.901388:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099779584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776663.901394:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776663.901398:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18389) 00000800:00000200:2.1:1775776663.901562:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776663.901565:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776663.901937:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776663.901941:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776663.902295:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776663.902297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776663.902481:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776663.902482:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1775776664.603888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776664.603894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776664.603896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776664.603901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776664.603905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776664.603908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776664.603911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776664.603913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776664.603917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776664.667974:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776664.667978:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776664.667981:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776664.668017:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776664.668020:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776664.668022:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776664.668025:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776664.668028:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776664.668031:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776664.668034:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776664.668036:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776664.668038:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776665.179859:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313052917, at 4313052928 00000100:00000040:1.0:1775776665.179859:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313052917, at 4313052928 00000100:00000010:1.0:1775776665.179875:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000010:0.0:1775776665.179876:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000001:3.0:1775776665.627886:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776665.627892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776665.627893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776665.627898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776665.627902:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776665.627906:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776665.627908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776665.627910:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776665.627912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776665.692019:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776665.692022:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776665.692024:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776665.692070:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776665.692074:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776665.692076:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776665.692080:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776665.692091:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776665.692094:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776665.692096:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776665.692098:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776665.692101:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776666.651897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776666.651919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776666.651921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776666.651927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776666.651930:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776666.651934:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776666.651937:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776666.651939:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776666.651941:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776666.716270:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776666.716279:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776666.716281:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776666.716286:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776666.716290:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776666.716293:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776666.716296:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776666.716298:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776666.716300:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776666.717035:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776666.717039:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776666.717042:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776667.677038:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776667.677046:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776667.677047:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776667.677052:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776667.677056:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776667.677060:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776667.677063:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776667.677065:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776667.677068:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776667.740758:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776667.740773:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776667.740776:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776667.740802:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776667.740806:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776667.740810:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776667.740812:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776667.740814:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776667.740817:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776667.741767:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776667.741771:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776667.741774:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776668.699901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776668.699908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776668.699909:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776668.699913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776668.699916:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776668.699919:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776668.699921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776668.699923:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776668.699926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776668.763881:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776668.763889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776668.763890:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776668.763895:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776668.763899:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776668.763903:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776668.763905:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776668.763907:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776668.763910:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776668.764059:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776668.764063:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776668.764067:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776669.019938:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776669.019947:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776669.019950:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776669.019954:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776669.019958:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776669.019965:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18384 00000400:00000200:3.0:1775776669.019975:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776669.019980:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776669.019983:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776669.019986:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.019990:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.019992:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.019995:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776669.019998:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776669.020000:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776669.020003:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776669.020006:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776669.020008:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776669.020010:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776669.020013:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020015:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020017:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020019:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776669.020021:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776669.020023:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020026:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776669.020028:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776669.020031:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776669.020034:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020036:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776669.020038:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776669.020046:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775776669.020050:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776669.020053:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776669.020055:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020058:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776669.020060:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776669.020063:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776669.020070:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776669.020076:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776669.020084:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099779712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776669.020100:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776669.020104:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776669.020109:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099779712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776669.020129:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776669.020132:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776669.020135:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776669.020137:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776669.020138:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776669.020141:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776669.020143:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18384 00000400:00000200:3.0:1775776669.020146:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776669.020150:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776669.020153:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776669.020156:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020158:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020163:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020166:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776669.020169:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776669.020171:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776669.020173:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776669.020176:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776669.020181:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776669.020185:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776669.020191:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020194:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020198:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020201:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776669.020205:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776669.020210:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020216:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776669.020218:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776669.020224:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776669.020227:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020230:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776669.020235:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776669.020238:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020241:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776669.020244:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000020:00000040:3.0:1775776669.020246:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776669.020249:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776669.020250:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020254:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776669.020255:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776669.020257:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776669.020263:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776669.020267:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776669.020274:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099779840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776669.020284:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776669.020286:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776669.020292:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099779840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776669.020306:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776669.020311:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776669.020316:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776669.020319:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776669.020321:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776669.020326:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776669.020331:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18384 00000400:00000200:3.0:1775776669.020337:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776669.020342:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776669.020344:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776669.020346:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020349:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020353:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020356:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776669.020359:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776669.020361:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776669.020364:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776669.020367:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776669.020369:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776669.020373:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776669.020378:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020380:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020384:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020387:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776669.020390:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776669.020395:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020399:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776669.020403:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776669.020406:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776669.020411:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020413:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776669.020415:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776669.020419:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020422:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776669.020428:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776669.020431:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776669.020435:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776669.020437:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020438:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776669.020440:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776669.020441:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776669.020444:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776669.020448:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776669.020454:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099779968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776669.020465:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776669.020469:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776669.020475:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099779968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776669.020481:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776669.020487:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1775776669.020489:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776669.020493:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776669.020499:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776669.020502:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776669.020505:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776669.020507:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776669.020511:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18384 00000400:00000200:3.0:1775776669.020517:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776669.020523:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776669.020526:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776669.020530:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020534:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020537:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020540:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776669.020543:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776669.020545:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776669.020548:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776669.020550:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776669.020554:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776669.020559:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776669.020564:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020567:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020571:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020576:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776669.020582:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776669.020587:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020590:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776669.020592:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776669.020594:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776669.020595:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020597:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776669.020599:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776669.020601:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020603:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776669.020609:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776669.020614:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776669.020618:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776669.020620:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020622:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776669.020624:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776669.020625:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776669.020632:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776669.020635:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776669.020641:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099780096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776669.020649:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776669.020651:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776669.020657:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099780096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776669.020669:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776669.020674:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776669.020680:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776669.020682:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776669.020684:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776669.020689:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776669.020693:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18384 00000400:00000200:3.0:1775776669.020700:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776669.020705:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776669.020708:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776669.020712:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020716:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020719:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020724:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776669.020728:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776669.020730:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776669.020733:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776669.020735:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776669.020740:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776669.020744:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776669.020750:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020754:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776669.020758:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020762:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776669.020821:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776669.020828:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776669.020834:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776669.020837:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776669.020840:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776669.020844:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020848:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776669.020852:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776669.020856:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020859:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776669.020876:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776669.020881:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776669.020885:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776669.020888:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.020893:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776669.020895:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776669.020899:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776669.020905:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776669.020910:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776669.020919:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099780224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776669.020931:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776669.020934:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776669.020940:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099780224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776669.020952:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776669.020958:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18394) 00000800:00000200:2.1:1775776669.021120:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776669.021122:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776669.021350:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776669.021351:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776669.021601:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776669.021603:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776669.021876:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776669.021878:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1775776669.723866:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776669.723871:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776669.723873:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776669.723878:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776669.723882:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776669.723885:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776669.723887:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776669.723890:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776669.723892:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776669.787863:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776669.787870:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776669.787872:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776669.787879:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776669.787883:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776669.787887:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776669.787890:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776669.787893:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776669.787895:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776669.792029:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776669.792032:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776669.792036:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776670.299932:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313058037, at 4313058048 00000100:00000040:1.0:1775776670.299932:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313058037, at 4313058048 00000100:00000010:1.0:1775776670.299950:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000010:0.0:1775776670.299952:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:3.0:1775776670.747966:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776670.747974:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776670.747976:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776670.747983:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776670.747988:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776670.747992:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776670.747995:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776670.747998:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776670.748001:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776670.811968:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776670.811979:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776670.811983:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776670.811992:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776670.812001:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776670.812008:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776670.812013:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776670.812017:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776670.812022:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776670.812277:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776670.812282:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776670.812286:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776671.771925:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776671.771933:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776671.771935:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776671.771941:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776671.771944:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776671.771948:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776671.771950:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776671.771952:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776671.771955:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776671.835868:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776671.835875:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776671.835880:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776671.835887:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776671.835891:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776671.835896:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776671.835902:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776671.835906:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776671.835912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776671.835979:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776671.835982:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776671.835985:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776672.795902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776672.795910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776672.795913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776672.795918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776672.795921:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776672.795925:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776672.795927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776672.795929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776672.795932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776672.859894:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776672.859901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776672.859903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776672.859910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776672.859915:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776672.859920:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776672.859923:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776672.859927:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776672.859932:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776672.860024:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776672.860028:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776672.860031:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776673.819943:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776673.819950:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776673.819951:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776673.819956:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776673.819972:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776673.819977:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776673.819979:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776673.819982:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776673.819984:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776673.883888:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776673.883894:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776673.883895:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776673.883899:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776673.883903:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776673.883905:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776673.883907:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776673.883909:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776673.883912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776673.884080:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776673.884084:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776673.884087:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776674.140037:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776674.140046:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776674.140049:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776674.140053:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776674.140057:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776674.140064:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18389 00000400:00000200:1.0:1775776674.140073:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776674.140079:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776674.140082:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776674.140085:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140088:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140090:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140092:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776674.140096:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776674.140097:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776674.140100:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776674.140102:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776674.140104:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776674.140106:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776674.140109:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140110:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140112:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140114:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776674.140117:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776674.140120:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140122:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776674.140125:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776674.140127:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776674.140129:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140131:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776674.140133:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776674.140153:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776674.140157:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776674.140160:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776674.140162:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140164:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776674.140167:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776674.140170:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776674.140177:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776674.140182:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776674.140190:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099780352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776674.140201:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776674.140205:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776674.140209:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099780352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776674.140231:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776674.140234:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776674.140237:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776674.140238:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776674.140240:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776674.140242:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776674.140244:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18389 00000400:00000200:1.0:1775776674.140248:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776674.140251:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776674.140253:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776674.140256:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140258:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140261:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140263:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776674.140265:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776674.140267:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776674.140269:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776674.140272:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776674.140274:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776674.140277:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776674.140282:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140284:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140289:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140294:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776674.140298:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776674.140302:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140306:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776674.140309:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776674.140314:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776674.140318:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140322:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776674.140327:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776674.140333:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140336:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776674.140339:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776674.140343:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776674.140347:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776674.140349:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140352:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776674.140353:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776674.140354:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776674.140359:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776674.140363:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776674.140371:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099780480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776674.140383:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776674.140387:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776674.140391:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099780480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776674.140409:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776674.140413:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776674.140420:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776674.140423:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776674.140426:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776674.140430:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776674.140437:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18389 00000400:00000200:1.0:1775776674.140443:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776674.140447:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776674.140451:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776674.140454:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140457:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140461:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140464:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776674.140466:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776674.140467:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776674.140470:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776674.140473:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776674.140476:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776674.140480:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776674.140485:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140488:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140492:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140495:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776674.140499:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776674.140504:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140512:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776674.140515:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776674.140519:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776674.140522:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140526:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776674.140529:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776674.140531:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140534:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776674.140539:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776674.140543:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776674.140547:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776674.140548:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140550:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776674.140551:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776674.140553:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776674.140556:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776674.140560:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776674.140566:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099780608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776674.140577:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776674.140580:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776674.140586:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099780608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776674.140593:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776674.140599:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:1.0:1775776674.140600:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776674.140603:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776674.140609:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776674.140612:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776674.140614:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776674.140618:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776674.140622:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18389 00000400:00000200:1.0:1775776674.140626:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776674.140632:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776674.140635:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776674.140638:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140641:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140645:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140648:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776674.140651:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776674.140654:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776674.140656:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776674.140658:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776674.140662:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776674.140666:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776674.140672:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140675:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140679:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140682:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776674.140685:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776674.140691:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140696:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776674.140699:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776674.140702:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776674.140706:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140709:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776674.140712:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776674.140717:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140720:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776674.140723:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776674.140726:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776674.140730:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776674.140731:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140734:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776674.140736:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776674.140738:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776674.140742:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776674.140745:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776674.140750:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099780736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776674.140760:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776674.140763:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776674.140768:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099780736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776674.140815:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776674.140819:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776674.140822:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776674.140823:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776674.140825:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776674.140827:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776674.140830:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18389 00000400:00000200:1.0:1775776674.140834:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776674.140838:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776674.140841:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776674.140844:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140848:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140851:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140854:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776674.140856:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776674.140858:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776674.140860:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776674.140861:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776674.140863:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776674.140865:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776674.140868:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140871:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776674.140874:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140876:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776674.140879:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776674.140884:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776674.140889:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776674.140892:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776674.140895:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776674.140897:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140900:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776674.140903:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776674.140909:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140911:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776674.140916:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776674.140920:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776674.140924:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776674.140926:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776674.140930:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776674.140932:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776674.140934:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776674.140939:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776674.140943:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776674.140951:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099780864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776674.140962:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776674.140965:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776674.140971:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099780864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776674.140978:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776674.140983:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18399) 00000800:00000200:2.1:1775776674.141279:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776674.141281:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776674.141572:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776674.141574:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776674.141866:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776674.141868:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776674.142126:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776674.142129:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1775776674.843897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776674.843905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776674.843907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776674.843913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776674.843916:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776674.843920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776674.843922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776674.843924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776674.843927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776674.908006:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776674.908010:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776674.908013:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776674.908479:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776674.908483:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776674.908485:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776674.908489:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776674.908492:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776674.908495:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776674.908498:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776674.908500:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776674.908503:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776675.419909:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313063158, at 4313063168 00000100:00000010:1.0:1775776675.419926:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000040:0.0:1775776675.420034:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313063157, at 4313063168 00000100:00000010:0.0:1775776675.420040:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000001:3.0:1775776675.868158:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776675.868165:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776675.868167:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776675.868172:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776675.868175:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776675.868179:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776675.868181:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776675.868183:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776675.868185:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776675.931855:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776675.931862:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776675.931863:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776675.931868:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776675.931872:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776675.931875:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776675.931878:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776675.931880:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776675.931882:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776675.931989:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776675.931993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776675.931996:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776676.891952:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776676.891967:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776676.891970:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776676.891975:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776676.891979:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776676.891982:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776676.891985:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776676.891987:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776676.891990:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776676.955940:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776676.955946:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776676.955949:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776676.955954:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776676.955959:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776676.955965:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776676.955969:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776676.955972:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776676.955975:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776676.959166:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776676.959172:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776676.959177:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776677.917046:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776677.917054:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776677.917055:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776677.917061:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776677.917064:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776677.917068:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776677.917071:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776677.917073:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776677.917075:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776677.980043:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776677.980046:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776677.980049:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776677.980553:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776677.980569:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776677.980571:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776677.980576:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776677.980581:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776677.980585:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776677.980588:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776677.980591:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776677.980595:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776678.939865:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776678.939872:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776678.939874:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776678.939880:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776678.939884:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776678.939888:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776678.939890:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776678.939891:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776678.939893:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776679.004006:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776679.004013:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776679.004015:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776679.004021:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776679.004025:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776679.004028:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776679.004031:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776679.004034:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776679.004036:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776679.004353:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776679.004358:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776679.004362:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776679.260020:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776679.260031:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776679.260034:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776679.260039:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776679.260044:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776679.260053:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18394 00000400:00000200:0.0:1775776679.260063:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776679.260071:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776679.260075:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776679.260078:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260082:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.260085:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260087:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776679.260092:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776679.260094:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776679.260098:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776679.260105:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776679.260107:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776679.260110:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776679.260113:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260115:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.260117:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260118:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776679.260121:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776679.260124:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260127:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776679.260130:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776679.260134:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776679.260141:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.260144:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776679.260147:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776679.260158:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776679.260163:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776679.260168:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776679.260170:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.260173:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776679.260177:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776679.260182:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776679.260192:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776679.260199:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776679.260215:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099780992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776679.260232:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776679.260238:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776679.260244:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099780992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776679.260487:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776679.260489:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1775776679.260636:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776679.260639:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776679.260642:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776679.260643:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776679.260645:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776679.260647:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776679.260649:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18394 00000400:00000200:0.0:1775776679.260652:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776679.260654:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776679.260656:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776679.260658:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260660:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.260663:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260665:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776679.260667:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776679.260668:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776679.260670:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776679.260671:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776679.260673:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776679.260675:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776679.260678:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260680:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.260682:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260710:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776679.260713:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776679.260715:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776679.260718:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776679.260719:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776679.260726:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776679.260728:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.260730:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776679.260735:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776679.260738:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.260740:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776679.260744:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775776679.260746:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776679.260749:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776679.260750:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.260752:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776679.260753:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776679.260755:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776679.260759:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776679.260763:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776679.260768:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099781120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776679.260776:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776679.260811:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776679.260816:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099781120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1775776679.261170:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776679.261173:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1775776679.261412:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776679.261414:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776679.261417:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776679.261418:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776679.261422:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776679.261424:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776679.261426:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18394 00000400:00000200:0.0:1775776679.261429:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776679.261432:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776679.261433:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776679.261435:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261437:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.261439:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261441:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776679.261443:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776679.261444:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776679.261445:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776679.261447:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776679.261449:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776679.261451:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776679.261456:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261457:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.261459:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261461:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776679.261462:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776679.261465:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261466:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776679.261467:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776679.261469:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776679.261470:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261471:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776679.261473:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776679.261474:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261475:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776679.261479:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775776679.261481:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776679.261483:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776679.261484:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261485:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776679.261487:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776679.261488:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776679.261491:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776679.261494:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776679.261497:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099781248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776679.261503:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776679.261506:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776679.261509:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099781248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776679.261524:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776679.261527:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776679.261530:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776679.261531:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776679.261533:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776679.261535:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776679.261539:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18394 00000400:00000200:0.0:1775776679.261543:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776679.261547:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776679.261551:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776679.261557:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261561:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.261566:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261571:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776679.261576:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776679.261578:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776679.261581:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776679.261584:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776679.261588:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776679.261593:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776679.261600:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261605:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.261611:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261614:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776679.261617:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776679.261623:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261626:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776679.261630:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776679.261634:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776679.261639:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261642:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776679.261647:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776679.261653:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261655:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776679.261661:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776679.261665:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776679.261669:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776679.261672:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261676:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776679.261678:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776679.261680:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776679.261696:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776679.261701:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776679.261710:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099781376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776679.261724:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776679.261728:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776679.261736:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099781376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776679.261753:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776679.261758:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776679.261765:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776679.261768:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776679.261771:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776679.261776:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776679.261805:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18394 00000400:00000200:0.0:1775776679.261815:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776679.261820:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776679.261822:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776679.261826:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261831:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.261835:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261840:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776679.261844:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776679.261846:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776679.261849:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776679.261853:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776679.261858:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000800:00000200:2.1:1775776679.261858:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776679.261863:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000020:00000001:0.0:1775776679.261864:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776679.261872:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261877:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776679.261882:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261886:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776679.261891:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776679.261897:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776679.261902:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776679.261906:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776679.261910:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776679.261914:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261919:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776679.261923:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776679.261926:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261928:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776679.261932:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776679.261936:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776679.261941:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776679.261944:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776679.261947:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776679.261950:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776679.261953:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776679.261958:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776679.261964:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776679.261974:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099781504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776679.261988:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776679.261993:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776679.262000:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099781504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776679.262016:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776679.262023:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18404) 00000800:00000200:2.1:1775776679.262304:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776679.262306:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776679.262572:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776679.262574:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1775776679.963931:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776679.963939:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776679.963941:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776679.963949:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776679.963953:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776679.963958:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776679.963960:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776679.963963:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776679.963966:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776680.027923:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776680.027931:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776680.027933:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776680.027940:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776680.027944:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776680.027949:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776680.027952:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776680.027954:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776680.027957:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776680.028694:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776680.028698:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776680.028701:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776680.539885:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313068278, at 4313068288 00000100:00000010:1.0:1775776680.539902:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000040:0.0:1775776680.540076:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313068278, at 4313068288 00000100:00000010:0.0:1775776680.540083:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000001:3.0:1775776680.988717:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776680.988724:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776680.988726:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776680.988732:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776680.988886:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776680.988892:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776680.988895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776680.988898:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776680.988901:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776681.051914:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776681.051923:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776681.051926:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776681.051938:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776681.051944:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776681.051950:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776681.051955:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776681.051960:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776681.051967:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776681.052063:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776681.052068:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776681.052072:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776682.011951:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776682.011957:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776682.011959:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776682.011964:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776682.011967:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776682.011972:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776682.011974:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776682.011977:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776682.011979:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776682.075904:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776682.075912:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776682.075914:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776682.075921:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776682.075926:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776682.075933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776682.075938:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776682.075943:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776682.075948:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776682.076026:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776682.076029:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776682.076032:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776683.035951:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776683.035958:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776683.035960:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776683.035965:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776683.035968:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776683.035972:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776683.035975:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776683.035977:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776683.035980:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776683.099906:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776683.099915:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776683.099917:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776683.099923:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776683.099928:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776683.099933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776683.099936:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776683.099939:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776683.099942:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776683.100115:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776683.100118:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776683.100121:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776684.059911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776684.059916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776684.059918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776684.059922:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776684.059925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776684.059928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776684.059929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776684.059931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776684.059932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776684.123885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776684.123892:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776684.123894:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776684.123899:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776684.123902:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776684.123906:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776684.123908:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776684.123910:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776684.123912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776684.124456:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776684.124461:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776684.124464:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776684.379904:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776684.379914:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776684.379916:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776684.379921:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776684.379925:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776684.379933:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18399 00000400:00000200:0.0:1775776684.379947:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776684.379954:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776684.379957:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776684.379960:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.379963:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.379966:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.379968:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776684.379973:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776684.379974:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776684.379977:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776684.379979:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776684.379981:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776684.379983:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776684.379986:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.379987:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.379989:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.379991:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776684.379994:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776684.379997:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776684.379999:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776684.380002:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776684.380005:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776684.380008:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380010:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776684.380012:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776684.380020:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776684.380024:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776684.380027:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776684.380029:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380032:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776684.380034:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776684.380038:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776684.380046:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776684.380052:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776684.380061:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099781632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776684.380072:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776684.380076:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776684.380080:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099781632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776684.380111:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776684.380114:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776684.380117:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776684.380118:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776684.380119:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776684.380122:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776684.380124:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18399 00000400:00000200:0.0:1775776684.380128:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776684.380131:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776684.380133:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776684.380135:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380137:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.380139:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380141:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776684.380143:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776684.380145:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776684.380146:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776684.380148:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776684.380150:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776684.380152:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776684.380154:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380156:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.380158:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380160:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776684.380163:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776684.380165:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380167:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776684.380169:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776684.380173:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776684.380175:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380177:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776684.380181:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776684.380184:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380186:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776684.380189:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776684.380192:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776684.380194:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776684.380195:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380197:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776684.380198:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776684.380200:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776684.380203:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776684.380206:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776684.380210:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099781760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776684.380216:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776684.380218:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776684.380221:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099781760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776684.380227:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776684.380230:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776684.380233:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776684.380234:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776684.380236:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776684.380238:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776684.380240:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18399 00000400:00000200:0.0:1775776684.380243:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776684.380246:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776684.380247:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776684.380260:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380262:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.380264:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380266:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776684.380268:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776684.380269:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776684.380270:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776684.380271:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776684.380273:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776684.380275:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776684.380277:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380279:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.380280:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380282:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776684.380284:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776684.380286:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380288:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776684.380290:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776684.380292:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776684.380294:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380296:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776684.380298:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776684.380300:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380301:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776684.380305:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776684.380307:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776684.380309:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776684.380310:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380311:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776684.380312:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776684.380314:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776684.380324:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776684.380326:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776684.380330:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099781888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776684.380337:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776684.380339:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776684.380342:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099781888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776684.380349:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776684.380351:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776684.380354:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776684.380355:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776684.380356:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776684.380358:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776684.380365:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18399 00000400:00000200:0.0:1775776684.380369:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776684.380372:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776684.380374:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776684.380388:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380390:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.380392:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380394:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776684.380396:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776684.380397:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776684.380398:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776684.380400:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776684.380402:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776684.380404:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776684.380407:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380409:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.380410:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380412:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776684.380415:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776684.380417:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380419:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776684.380421:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776684.380423:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776684.380425:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380427:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776684.380430:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776684.380432:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380433:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776684.380437:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776684.380439:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776684.380441:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776684.380442:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380443:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776684.380445:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776684.380446:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776684.380449:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776684.380452:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776684.380456:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099782016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776684.380462:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776684.380464:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776684.380468:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099782016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776684.380475:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776684.380477:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776684.380480:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776684.380482:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776684.380483:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776684.380485:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776684.380487:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18399 00000400:00000200:0.0:1775776684.380490:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776684.380493:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776684.380495:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776684.380497:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380499:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.380501:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380502:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776684.380504:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776684.380505:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776684.380507:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776684.380508:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776684.380510:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776684.380512:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776684.380515:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380517:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776684.380518:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380520:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776684.380521:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776684.380524:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776684.380526:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776684.380527:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776684.380530:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776684.380532:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380533:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776684.380536:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776684.380537:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380539:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776684.380542:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776684.380544:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776684.380546:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776684.380548:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776684.380549:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776684.380550:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776684.380551:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776684.380554:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776684.380557:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776684.380561:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099782144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776684.380567:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776684.380569:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776684.380572:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099782144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776684.380578:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776684.380582:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18409) 00000800:00000200:2.1:1775776684.380967:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776684.380970:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776684.381294:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776684.381297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776684.381552:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776684.381555:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776684.381859:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776684.381862:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776684.382119:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776684.382121:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1775776685.083883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776685.083887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776685.083889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776685.083893:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776685.083895:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776685.083898:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776685.083900:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776685.083902:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776685.083904:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776685.147888:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776685.147895:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776685.147899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776685.147905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776685.147911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776685.147916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776685.147920:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776685.147923:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776685.147926:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776685.147979:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776685.147983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776685.147987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776685.659968:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313073398, at 4313073408 00000100:00000010:0.0:1775776685.659985:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000040:1.0:1775776685.660018:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313073398, at 4313073408 00000100:00000010:1.0:1775776685.660033:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000001:3.0:1775776686.108037:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776686.108042:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776686.108044:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776686.108049:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776686.108053:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776686.108056:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776686.108058:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776686.108060:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776686.108063:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776686.171876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776686.171883:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776686.171886:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776686.171892:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776686.171897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776686.171902:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776686.171906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776686.171908:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776686.171911:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776686.171979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776686.171984:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776686.171987:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776687.131900:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776687.131906:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776687.131908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776687.131913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776687.131917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776687.131921:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776687.131923:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776687.131925:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776687.131928:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776687.195876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776687.195882:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776687.195887:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776687.195892:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776687.195897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776687.195900:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776687.195905:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776687.195907:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776687.195910:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776687.195966:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776687.195969:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776687.195972:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776688.155888:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776688.155893:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776688.155895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776688.155901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776688.155904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776688.155908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776688.155911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776688.155912:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776688.155915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776688.219906:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776688.219914:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776688.219917:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776688.219924:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776688.219929:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776688.219933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776688.219936:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776688.219940:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776688.219944:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776688.220040:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776688.220043:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776688.220046:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776689.179918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776689.179925:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776689.179927:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776689.179932:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776689.179936:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776689.179940:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776689.179942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776689.179944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776689.179947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.243891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776689.243899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776689.243902:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776689.243908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776689.243912:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776689.243918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776689.243923:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776689.243925:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776689.243930:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776689.243996:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776689.244000:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776689.244003:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776689.499915:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776689.499933:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776689.499936:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776689.499941:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776689.499946:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776689.499953:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18404 00000400:00000200:3.0:1775776689.499963:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776689.499969:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776689.499973:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776689.499976:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.499979:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.499981:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.499984:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776689.499988:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776689.499990:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776689.499993:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776689.499996:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776689.499998:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776689.500011:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776689.500014:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500016:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500018:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500019:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776689.500022:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776689.500025:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500028:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776689.500030:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776689.500034:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776689.500036:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500038:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776689.500040:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776689.500049:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776689.500052:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776689.500055:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776689.500057:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500060:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776689.500062:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776689.500065:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776689.500072:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776689.500078:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776689.500087:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099782272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776689.500098:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776689.500102:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776689.500107:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099782272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776689.500128:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776689.500130:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776689.500133:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776689.500134:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776689.500135:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776689.500137:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776689.500138:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18404 00000400:00000200:3.0:1775776689.500141:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776689.500143:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776689.500144:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776689.500146:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500147:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500149:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500150:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776689.500152:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776689.500152:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776689.500154:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776689.500155:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776689.500156:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776689.500158:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776689.500160:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500161:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500162:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500163:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776689.500164:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776689.500166:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500168:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776689.500169:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776689.500173:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776689.500175:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500176:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776689.500179:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776689.500182:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500183:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776689.500187:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776689.500189:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776689.500192:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776689.500195:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500198:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776689.500200:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776689.500202:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776689.500206:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776689.500211:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776689.500218:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099782400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776689.500233:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776689.500237:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776689.500246:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099782400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776689.500271:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776689.500275:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776689.500280:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776689.500281:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776689.500283:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776689.500288:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776689.500290:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18404 00000400:00000200:3.0:1775776689.500295:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776689.500299:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776689.500304:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776689.500309:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500311:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500315:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500320:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776689.500325:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776689.500328:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776689.500331:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776689.500333:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776689.500337:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776689.500340:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776689.500346:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500350:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500354:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500356:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776689.500361:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776689.500366:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500370:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776689.500374:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776689.500377:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776689.500380:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500382:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776689.500385:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776689.500388:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500390:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776689.500394:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776689.500399:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776689.500403:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776689.500405:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500408:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776689.500410:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776689.500412:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776689.500414:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775776689.500417:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776689.500423:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099782528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776689.500444:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776689.500446:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776689.500452:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099782528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776689.500466:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776689.500471:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:3.0:1775776689.500472:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1775776689.500477:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00080000:3.0:1775776689.500478:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776689.500481:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776689.500484:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776689.500489:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776689.500494:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18404 00000400:00000200:3.0:1775776689.500500:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776689.500519:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776689.500524:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776689.500529:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500533:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500537:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500540:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776689.500544:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776689.500546:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776689.500549:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776689.500552:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776689.500557:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776689.500562:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776689.500567:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500571:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500575:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500578:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776689.500582:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776689.500586:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500589:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776689.500591:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776689.500594:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776689.500597:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500598:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776689.500600:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776689.500603:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500604:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776689.500608:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776689.500611:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776689.500613:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776689.500614:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500615:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776689.500616:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776689.500627:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776689.500635:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776689.500640:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776689.500646:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099782656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776689.500658:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776689.500662:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776689.500669:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099782656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776689.500682:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776689.500687:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776689.500696:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776689.500700:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776689.500703:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776689.500706:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776689.500710:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18404 00000400:00000200:3.0:1775776689.500716:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776689.500723:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776689.500727:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776689.500732:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500735:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500739:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500743:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776689.500746:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776689.500748:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776689.500750:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776689.500752:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776689.500754:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776689.500757:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776689.500763:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500767:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776689.500772:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500775:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776689.500811:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776689.500814:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776689.500817:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776689.500821:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776689.500824:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776689.500828:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500830:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776689.500833:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776689.500837:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500840:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776689.500842:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776689.500844:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776689.500845:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776689.500847:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776689.500851:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776689.500853:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776689.500855:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776689.500859:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776689.500863:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776689.500871:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099782784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776689.500879:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776689.500882:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776689.500885:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099782784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776689.500895:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776689.500901:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18414) 00000800:00000200:2.1:1775776689.501110:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776689.501113:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776689.501344:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776689.501346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776689.501589:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776689.501592:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776689.501871:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776689.501874:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1775776690.203892:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776690.203899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776690.203901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776690.203906:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776690.203910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776690.203913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776690.203916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776690.203919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776690.203922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776690.267874:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776690.267891:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776690.267894:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776690.267900:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776690.267906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776690.267910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776690.267912:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776690.267914:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776690.267916:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776690.267969:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776690.267971:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776690.267973:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776690.780464:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313078517, at 4313078528 00000100:00000010:0.0:1775776690.780477:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000040:1.0:1775776690.781827:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313078517, at 4313078530 00000100:00000010:1.0:1775776690.781835:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000001:3.0:1775776691.228601:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776691.228608:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776691.228610:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776691.228615:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776691.228619:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776691.228622:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776691.228625:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776691.228627:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776691.228629:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776691.291891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776691.291899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776691.291902:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776691.291908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776691.291912:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776691.291917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776691.291921:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776691.291924:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776691.291928:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776691.292050:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776691.292053:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776691.292065:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776692.251881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776692.251887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776692.251889:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776692.251892:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776692.251895:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776692.251898:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776692.251900:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776692.251902:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776692.251903:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.315891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776692.315899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776692.315903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776692.315908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776692.315914:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776692.315918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776692.315922:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776692.315925:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776692.315932:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776692.315994:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776692.315998:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776692.316010:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776693.275895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776693.275901:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776693.275902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776693.275907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776693.275911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776693.275914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776693.275917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776693.275919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776693.275921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776693.340052:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776693.340057:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776693.340059:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776693.340063:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776693.340066:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776693.340070:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776693.340072:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776693.340075:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776693.340077:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776693.340733:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776693.340751:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776693.340754:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776694.299998:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776694.300004:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776694.300006:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776694.300010:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776694.300013:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776694.300016:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776694.300018:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776694.300019:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776694.300021:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776694.363876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776694.363894:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776694.363897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776694.363904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776694.363909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776694.363912:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776694.363915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776694.363917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776694.363919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776694.363994:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776694.363996:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776694.363999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776694.619959:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776694.619969:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776694.619972:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776694.619976:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776694.619980:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776694.619987:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18409 00000400:00000200:0.0:1775776694.620007:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776694.620013:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776694.620017:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776694.620020:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620023:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620025:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620028:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776694.620032:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776694.620034:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776694.620036:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776694.620039:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776694.620041:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776694.620043:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776694.620045:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620047:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620049:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620051:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776694.620053:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776694.620056:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620059:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776694.620062:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776694.620065:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776694.620068:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620070:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776694.620072:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776694.620081:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776694.620084:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776694.620088:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776694.620090:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620092:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776694.620106:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776694.620110:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776694.620118:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776694.620124:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776694.620133:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099782912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776694.620143:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776694.620147:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776694.620152:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099782912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776694.620177:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776694.620180:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776694.620184:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776694.620185:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776694.620187:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776694.620189:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776694.620192:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18409 00000400:00000200:0.0:1775776694.620196:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776694.620199:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776694.620200:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776694.620202:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620203:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620205:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620206:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776694.620207:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776694.620208:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776694.620209:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776694.620210:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776694.620211:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776694.620212:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776694.620214:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620215:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620217:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620218:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776694.620219:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776694.620221:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620222:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776694.620223:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776694.620227:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776694.620228:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620229:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776694.620232:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776694.620234:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620235:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776694.620237:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776694.620239:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776694.620240:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776694.620241:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620242:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776694.620243:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776694.620244:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776694.620247:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776694.620249:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776694.620252:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099783040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776694.620257:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776694.620258:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776694.620260:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099783040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776694.620265:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776694.620267:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776694.620269:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776694.620269:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776694.620271:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776694.620272:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776694.620274:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18409 00000400:00000200:0.0:1775776694.620276:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776694.620277:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776694.620278:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776694.620280:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620281:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620282:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620283:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776694.620284:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776694.620284:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776694.620285:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776694.620286:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776694.620287:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776694.620288:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776694.620290:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620291:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620292:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620293:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776694.620294:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776694.620296:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620297:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776694.620298:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776694.620299:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776694.620300:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620301:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776694.620302:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776694.620303:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620304:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776694.620306:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776694.620307:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776694.620308:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776694.620309:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620310:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776694.620311:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776694.620312:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776694.620314:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776694.620315:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776694.620317:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099783168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776694.620321:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776694.620322:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776694.620324:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099783168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776694.620328:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776694.620329:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776694.620331:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776694.620332:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776694.620332:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776694.620334:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776694.620335:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18409 00000400:00000200:0.0:1775776694.620336:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776694.620338:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776694.620339:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776694.620340:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620341:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620342:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620343:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776694.620344:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776694.620345:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776694.620346:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776694.620347:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776694.620348:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776694.620349:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776694.620351:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620352:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620353:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620354:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776694.620355:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776694.620356:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620357:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776694.620358:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776694.620359:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776694.620360:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620361:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776694.620362:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776694.620364:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620364:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776694.620366:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776694.620367:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776694.620368:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776694.620369:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620369:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776694.620370:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776694.620371:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776694.620372:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776694.620374:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776694.620376:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099783296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776694.620379:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776694.620380:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776694.620382:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099783296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776694.620386:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776694.620388:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776694.620389:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776694.620390:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776694.620391:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776694.620392:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776694.620393:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18409 00000400:00000200:0.0:1775776694.620395:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776694.620396:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776694.620397:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776694.620398:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620399:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620400:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620401:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776694.620402:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776694.620403:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776694.620403:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776694.620404:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776694.620405:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776694.620406:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776694.620408:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620408:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776694.620409:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620410:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776694.620411:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776694.620413:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776694.620414:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776694.620445:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776694.620448:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776694.620451:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620453:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776694.620455:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776694.620457:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620459:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776694.620463:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776694.620467:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776694.620470:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776694.620472:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776694.620485:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776694.620491:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776694.620495:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776694.620502:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776694.620507:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776694.620515:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099783424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776694.620529:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776694.620533:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776694.620542:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099783424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776694.620556:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776694.620563:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18420) 00000800:00000200:2.1:1775776694.620683:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776694.620686:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775776694.620931:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776694.620932:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775776694.621121:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776694.621123:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775776694.621368:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776694.621370:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1775776694.621771:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776694.621773:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 11 secs later) 00000100:00000001:3.0:1775776695.323943:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776695.323948:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776695.323949:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776695.323953:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776695.323956:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776695.323959:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776695.323961:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776695.323964:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776695.323966:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776695.388045:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776695.388049:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776695.388051:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776695.388055:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776695.388059:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776695.388062:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776695.388065:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775776695.388068:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000001:3.0:1775776695.388068:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776695.388072:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776695.388073:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776695.388078:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776695.899875:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313083637, at 4313083648 00000100:00000010:1.0:1775776695.899889:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000040:0.0:1775776695.899978:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313083637, at 4313083648 00000100:00000010:0.0:1775776695.899984:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:3.0:1775776696.347885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776696.347892:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776696.347893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776696.347900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776696.347903:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776696.347907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776696.347909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776696.347912:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776696.347914:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776696.411985:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776696.411989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776696.411992:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776696.412030:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776696.412035:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776696.412037:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776696.412040:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776696.412043:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776696.412046:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776696.412049:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776696.412051:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776696.412053:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776697.373592:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776697.373599:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776697.373601:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776697.373606:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776697.373610:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776697.373614:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776697.373616:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776697.373619:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776697.373621:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776697.435887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776697.435894:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776697.435897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776697.435903:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776697.435907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776697.435912:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776697.435916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776697.435920:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776697.435923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776697.435974:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776697.435976:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776697.435979:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776698.395844:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776698.395852:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776698.395854:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776698.395859:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776698.395863:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776698.395866:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776698.395868:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776698.395872:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776698.395874:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776698.460207:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776698.460209:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776698.460211:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776698.460415:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776698.460420:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776698.460421:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776698.460425:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776698.460428:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776698.460432:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776698.460434:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776698.460437:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776698.460439:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776699.419908:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776699.419914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776699.419916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776699.419921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776699.419924:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776699.419928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776699.419930:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776699.419933:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776699.419935:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776699.484022:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776699.484029:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776699.484031:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776699.484036:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776699.484040:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776699.484045:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776699.484048:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776699.484051:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776699.484053:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776699.484155:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776699.484158:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776699.484162:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776699.740032:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776699.740042:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776699.740045:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776699.740049:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776699.740053:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776699.740060:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18415 00000400:00000200:1.0:1775776699.740069:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776699.740075:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776699.740078:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776699.740081:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740084:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.740086:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740088:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776699.740091:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776699.740093:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776699.740096:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776699.740099:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776699.740109:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776699.740111:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776699.740115:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740116:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.740118:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740120:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776699.740123:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776699.740126:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740129:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776699.740132:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776699.740135:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776699.740138:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740141:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776699.740143:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776699.740153:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776699.740156:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776699.740160:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776699.740161:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740164:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776699.740166:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776699.740170:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776699.740179:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776699.740185:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776699.740194:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099783552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776699.740204:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776699.740209:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776699.740214:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099783552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776699.740238:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776699.740242:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776699.740245:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776699.740246:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776699.740248:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776699.740250:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776699.740253:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18415 00000400:00000200:1.0:1775776699.740257:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776699.740259:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776699.740262:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776699.740264:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740266:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.740268:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740270:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776699.740272:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776699.740274:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776699.740275:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776699.740277:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776699.740280:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776699.740282:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776699.740286:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740288:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.740290:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740292:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776699.740294:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776699.740299:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740304:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776699.740307:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776699.740313:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776699.740318:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740321:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776699.740325:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776699.740341:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740344:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776699.740350:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776699.740355:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776699.740359:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776699.740361:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740364:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776699.740365:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776699.740369:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776699.740376:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776699.740382:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776699.740391:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099783680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776699.740403:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776699.740408:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776699.740415:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099783680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776699.740428:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776699.740431:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776699.740436:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776699.740437:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776699.740439:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776699.740442:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776699.740447:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18415 00000400:00000200:1.0:1775776699.740454:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776699.740460:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776699.740465:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776699.740471:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740475:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.740481:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740484:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776699.740487:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776699.740489:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776699.740492:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776699.740494:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776699.740505:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776699.740510:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776699.740517:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740522:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.740525:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740529:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776699.740533:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776699.740539:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740545:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776699.740549:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776699.740553:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776699.740557:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740561:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776699.740564:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776699.740574:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740579:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776699.740584:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776699.740589:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776699.740593:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776699.740596:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740599:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776699.740603:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776699.740606:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776699.740611:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000800:00000200:2.1:1775776699.740615:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776699.740617:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776699.740620:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000100:00100000:1.0:1775776699.740625:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099783808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776699.740639:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776699.740641:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776699.740649:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099783808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776699.740660:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776699.740663:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776699.740667:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776699.740671:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776699.740673:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776699.740677:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776699.740683:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18415 00000400:00000200:1.0:1775776699.740691:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776699.740708:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776699.740712:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776699.740716:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740721:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.740725:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740729:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776699.740733:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776699.740735:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776699.740737:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776699.740741:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776699.740743:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776699.740746:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776699.740751:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740753:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.740757:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740761:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776699.740776:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776699.740858:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776699.740863:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776699.740867:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776699.740870:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776699.740874:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740876:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776699.740879:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776699.740885:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740889:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776699.740894:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776699.740898:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776699.740903:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776699.740905:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.740907:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776699.740909:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776699.740911:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776699.740916:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776699.740922:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776699.740932:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099783936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776699.740949:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776699.740954:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776699.740960:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099783936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776699.740972:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776699.740975:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776699.740980:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776699.740983:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776699.740986:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776699.740990:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776699.740996:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18415 00000400:00000200:1.0:1775776699.741003:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776699.741010:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776699.741014:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776699.741020:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.741025:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.741028:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.741029:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776699.741034:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776699.741036:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776699.741038:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776699.741041:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776699.741044:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776699.741047:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776699.741054:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.741058:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776699.741060:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776699.741063:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776699.741066:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776699.741068:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776699.741072:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776699.741073:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776699.741076:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776699.741079:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.741086:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776699.741090:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776699.741092:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.741094:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776699.741099:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776699.741103:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776699.741108:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776699.741110:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776699.741113:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776699.741114:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776699.741116:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776699.741119:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776699.741125:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776699.741133:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099784064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776699.741154:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776699.741157:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776699.741162:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099784064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776699.741174:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776699.741179:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18425) 00000800:00000200:2.1:1775776699.741288:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776699.741290:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776699.741585:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776699.741587:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776699.741866:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776699.741869:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1775776699.742222:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776699.742224:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 6 secs later) 00000100:00000001:2.0:1775776699.742361:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776699.742364:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776699.742367:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776699.742369:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776699.742373:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ca80 x1862033099784064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776724 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776699.742380:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784064:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776699.742383:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776699.742384:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776699.742387:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ca80 x1862033099784064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776724 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776699.742392:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776699.742394:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776699.742395:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776699.742398:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776699.742399:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776699.742400:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776699.742402:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776699.742403:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38c00. 02000000:00000001:2.0:1775776699.742405:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776699.742406:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776699.742408:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ca80. 00000100:00000001:2.0:1775776699.742410:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776699.742411:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776699.742412:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776699.742415:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776699.742417:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776699.742419:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776699.742420:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776700.443903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776700.443910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776700.443911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776700.443918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776700.443922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776700.443926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776700.443929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776700.443931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776700.443934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776700.507899:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776700.507903:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776700.507906:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776700.507907:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776700.507909:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776700.507916:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776700.507917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776700.507919:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776700.507921:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776700.507926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776700.507926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776700.507929:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776700.507930:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776700.507934:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776700.507935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776700.507938:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776700.507939:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776700.508060:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776700.508063:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776700.508066:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776701.019897:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313088757, at 4313088768 00000100:00000010:1.0:1775776701.019912:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000040:0.0:1775776701.020012:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313088757, at 4313088768 00000100:00000010:0.0:1775776701.020021:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:3.0:1775776701.467924:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776701.467931:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776701.467933:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776701.467938:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776701.467941:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776701.467945:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776701.467947:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776701.467949:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776701.467952:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776701.532044:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776701.532051:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776701.532053:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776701.532058:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776701.532061:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776701.532065:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776701.532067:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776701.532070:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776701.532073:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532483:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776701.532486:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532491:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776701.532494:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776701.532497:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776701.532501:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532503:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776701.532506:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532508:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532510:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532514:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776701.532516:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776701.532517:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776701.532518:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532520:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776701.532521:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532523:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532524:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532528:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776701.532529:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776701.532530:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776701.532532:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532533:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776701.532534:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532535:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776701.532537:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776701.532679:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776701.532686:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776701.532689:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776702.492004:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776702.492019:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776702.492020:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776702.492025:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776702.492029:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776702.492032:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776702.492035:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776702.492037:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776702.492039:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776702.555887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776702.555894:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776702.555897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776702.555902:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776702.555906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776702.555910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776702.555912:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776702.555914:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776702.555917:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776702.555983:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776702.555987:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776702.555991:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776702.555992:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776702.555995:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776702.555997:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776702.556000:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776702.556001:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776702.556250:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776702.556254:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776702.556258:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776703.516093:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776703.516100:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776703.516101:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776703.516106:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776703.516109:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776703.516113:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776703.516123:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776703.516125:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776703.516128:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776703.579877:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776703.579882:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776703.579884:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776703.579886:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776703.579888:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776703.579891:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776703.579891:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776703.579896:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776703.579898:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776703.579899:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776703.579901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776703.579901:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776703.579905:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776703.579918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776703.579919:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776703.579923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776703.579925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776703.579979:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776703.579981:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776703.579984:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776704.539931:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776704.539937:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776704.539939:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776704.539945:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776704.539949:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776704.539953:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776704.539956:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776704.539958:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776704.539961:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776704.603931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776704.603939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776704.603945:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776704.603946:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776704.603950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776704.603951:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776704.603954:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776704.603955:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776704.604030:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776704.604033:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776704.604035:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776704.605326:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776704.605330:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776704.605332:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776704.605336:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776704.605341:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776704.605345:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776704.605347:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776704.605350:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776704.605352:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776704.859935:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776704.859942:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776704.859945:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776704.859948:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776704.859952:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776704.859963:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18420 00000400:00000200:0.0:1775776704.859970:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776704.859975:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776704.859977:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776704.859979:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.859982:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776704.859983:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.859985:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776704.859987:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776704.859989:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776704.859991:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776704.859992:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776704.859998:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776704.859999:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776704.860001:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860003:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776704.860004:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860005:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776704.860007:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776704.860009:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860010:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776704.860012:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776704.860014:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776704.860016:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860017:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776704.860019:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776704.860027:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776704.860029:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776704.860032:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776704.860033:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860036:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776704.860038:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776704.860041:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776704.860046:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776704.860051:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776704.860058:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776704.860064:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776704.860068:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776704.860071:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776704.860086:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776704.860088:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776704.860090:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776704.860094:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776704.860095:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776704.860097:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776704.860098:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18420 00000400:00000200:0.0:1775776704.860100:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776704.860102:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776704.860104:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776704.860106:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860107:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776704.860109:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860110:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776704.860112:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776704.860113:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776704.860114:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776704.860115:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776704.860117:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776704.860118:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776704.860120:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860122:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776704.860122:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776704.860126:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860127:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776704.860128:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1775776704.860129:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776704.860132:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776704.860132:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776704.860135:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776704.860137:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776704.860140:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776704.860142:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776704.860142:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776704.860145:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776704.860148:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776704.860152:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860155:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776704.860159:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000100:00000001:2.0:1775776704.860160:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1775776704.860162:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 00000100:00000001:2.0:1775776704.860163:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776704.860166:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776704.860168:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.860168:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775776704.860169:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776704.860171:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776704.860171:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776704.860174:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.860174:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776704.860178:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00100000:2.0:1775776704.860179:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099784192:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776704.860182:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.860183:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776704.860187:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1775776704.860188:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775776704.860190:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.860195:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776704.860198:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776704.860200:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776704.860200:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776704.860204:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776704.860206:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776704.860212:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776704.860217:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000100:00000001:0.0:1775776704.860219:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776704.860221:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1775776704.860223:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00080000:0.0:1775776704.860224:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776704.860227:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776704.860229:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776704.860230:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776704.860232:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18420 00000100:00000200:2.0:1775776704.860232:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099784192, portal 25 00000100:00000001:2.0:1775776704.860245:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1775776704.860248:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776704.860249:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757952 : -114841967793664 : ffff978d454dea00) 00000400:00000200:0.0:1775776704.860252:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776704.860254:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776704.860258:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000040:2.0:1775776704.860259:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776704.860260:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776704.860263:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860267:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776704.860271:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776704.860273:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776704.860276:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776704.860279:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000100:00000001:2.0:1775776704.860280:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1775776704.860281:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776704.860283:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776704.860287:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00000040:0.0:1775776704.860288:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860290:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776704.860291:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860292:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776704.860293:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776704.860295:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860297:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776704.860298:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1775776704.860298:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720dd48. 02000000:00000001:0.0:1775776704.860301:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1775776704.860303:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099784192, offset 0 02000000:00000001:0.0:1775776704.860304:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860307:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776704.860309:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776704.860313:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.860314:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860316:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776704.860320:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776704.860321:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776704.860325:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1775776704.860325:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:0.0:1775776704.860327:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860329:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776704.860330:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776704.860332:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.860334:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 02000000:00000010:0.0:1775776704.860335:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776704.860339:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.860341:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776704.860346:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776704.860350:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00100000:0.0:1775776704.860354:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776704.860356:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860363:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776704.860365:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776704.860372:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776704.860373:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4357:4357:256:4294967295] 192.168.201.139@tcp LPNI seq info [4357:4357:8:4294967295] 00000400:00000200:2.0:1775776704.860384:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776704.860392:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.860397:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00080000:0.0:1775776704.860398:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776704.860403:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776704.860405:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776704.860407:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.0:1775776704.860407:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860415:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000800:00000010:2.0:1775776704.860418:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7a00. 00000100:00080000:0.0:1775776704.860419:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18420 00000400:00000200:0.0:1775776704.860425:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776704.860429:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776704.860434:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1775776704.860434:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:0.0:1775776704.860439:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:2.0:1775776704.860440:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000100:00000001:0.0:1775776704.860443:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776704.860447:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776704.860449:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776704.860451:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000800:00000200:2.1:1775776704.860454:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000020:00000001:0.0:1775776704.860455:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776704.860458:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776704.860461:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000100:2.0:1775776704.860462:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000020:00000040:0.0:1775776704.860464:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000800:00000010:2.0:1775776704.860466:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000020:00000001:0.0:1775776704.860469:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776704.860471:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000400:00000200:2.0:1775776704.860473:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1775776704.860475:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860477:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776704.860481:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776704.860484:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775776704.860485:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776704.860488:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776704.860492:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860493:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.860493:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dd48 02000000:00000001:0.0:1775776704.860497:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1775776704.860498:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dd48. 02000000:00000001:0.0:1775776704.860499:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776704.860502:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860503:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776704.860504:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:0.0:1775776704.860506:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776704.860508:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860511:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776704.860512:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776704.860514:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776704.860517:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776704.860521:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776704.860523:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860526:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776704.860529:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776704.860530:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.860532:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000010:0.0:1775776704.860533:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776704.860536:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776704.860537:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776704.860543:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776704.860560:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.860564:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776704.860567:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776704.860569:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.860572:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776704.860573:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776704.860581:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776704.860582:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.860585:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313093877 00000100:00080000:0.0:1775776704.860586:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776704.860589:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776704.860590:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000010:2.0:1775776704.860591:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00080000:0.0:1775776704.860592:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776704.860595:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776704.860597:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18420 00000100:00000001:2.0:1775776704.860597:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000400:00000200:0.0:1775776704.860602:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776704.860604:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313093877 00000400:00000200:0.0:1775776704.860605:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000010:2.0:1775776704.860608:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bcc0. 00000100:00080000:0.0:1775776704.860609:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776704.860612:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1775776704.860613:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860616:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776704.860618:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776704.860618:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776704.860621:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860622:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776704.860625:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775776704.860625:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000020:00000040:0.0:1775776704.860627:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776704.860629:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776704.860630:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776704.860632:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00100000:2.0:1775776704.860632:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776704/real 1775776704] req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776704.860636:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776704.860642:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860645:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1775776704.860649:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776704.860650:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776704.860651:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776704.860653:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776704.860654:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000040:0.0:1775776704.860656:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776704.860657:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1775776704.860660:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000200:2.0:1775776704.860661:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776704.860664:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776704.860670:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776704.860674:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776704.860677:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776704.860680:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776704.860683:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776704.860684:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776704.860685:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776704.860687:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776704.860693:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860697:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.860701:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000010:0.0:1775776704.860702:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776704.860704:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775776704.860706:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:2.0:1775776704.860707:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1775776704.860711:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775776704.860711:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775776704.860714:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776704.860718:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1775776704.860718:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776704.860720:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776704.860723:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776704.860729:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:2.0:1775776704.860733:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776704.860734:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776704.860740:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776704.860742:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776704.860760:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776704.860762:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776704.860763:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776704.860764:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776704.860768:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.860777:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.860804:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.860807:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776704.860809:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776704.860811:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776704.860816:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776704.860819:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776704.860822:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776704.860824:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776704.860832:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18430) 00000100:00100000:2.0:1775776704.860832:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099784192:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776704.860837:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.860838:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.860842:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dea00 x1862033099784192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.860846:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776704.860848:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776704.860849:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776704.860852:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776704.860853:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776704.860855:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776704.860856:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776704.860858:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759cc00. 02000000:00000001:2.0:1775776704.860860:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776704.860862:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776704.860863:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dea00. 00000100:00000001:2.0:1775776704.860864:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776704.860865:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.860867:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.860872:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776704.860873:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.860874:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776704.860875:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776704.860876:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776704.860878:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.860891:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776704.860892:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.860894:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776704.860896:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776704.860898:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776704.860900:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784320:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776704.860903:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776704.860905:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776704.860906:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.860908:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776704.860911:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776704.860913:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.860918:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776704.860921:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776704.860924:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776704.860927:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099784320, portal 10 00000100:00000001:2.0:1775776704.860929:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776704.860931:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741752576 : -114841967799040 : ffff978d454dd500) 00000100:00000040:2.0:1775776704.860937:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.860944:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776704.860946:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776704.860948:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720dd48. 00000100:00000200:2.0:1775776704.860951:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099784320, offset 0 00000400:00000200:2.0:1775776704.860954:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776704.860963:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776704.860967:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776704.860970:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776704.860972:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776704.860974:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776704.860976:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776704.860979:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4358:4358:256:4294967295] 192.168.201.139@tcp LPNI seq info [4358:4358:8:4294967295] 00000400:00000200:2.0:1775776704.860983:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776704.860989:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776704.860993:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776704.860997:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7a00. 00000800:00000200:2.0:1775776704.861001:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776704.861004:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776704.861008:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776704.861016:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1775776704.861018:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776704.861019:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:2.0:1775776704.861021:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776704.861023:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776704.861025:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dd48 00000400:00000010:2.0:1775776704.861027:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dd48. 00000100:00000001:2.0:1775776704.861028:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776704.861031:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861035:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.861037:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861045:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861046:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776704.861048:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776704.861050:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776704.861052:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776704.861053:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313092613 00000100:00000001:2.0:1775776704.861055:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313093877 00000100:00000001:2.0:1775776704.861056:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313092613 00000100:00000001:2.0:1775776704.861058:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313093877 00000100:00000001:2.0:1775776704.861059:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776704.861060:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861061:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776704.861063:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776704/real 1775776704] req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776704.861067:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776704.861068:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776704.861069:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776704.861071:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776704.861071:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776704.861073:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776704.861074:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776704.861079:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861082:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776704.861083:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861084:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.861085:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776704.861086:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861090:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776704.861093:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861096:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776704.861098:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861098:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.861099:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861100:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776704.861101:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776704.861103:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776704.861104:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776704.861106:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776704.861107:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776704.861111:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784320:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776704.861113:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776704.861115:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861119:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776704.861120:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.861122:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776704.861123:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776704.861125:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776704.861128:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784448:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776704.861131:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776704.861133:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776704.861134:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.861136:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776704.861139:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776704.861141:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.861144:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776704.861146:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776704.861149:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776704.861151:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099784448, portal 10 00000100:00000001:2.0:1775776704.861153:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776704.861154:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741761536 : -114841967790080 : ffff978d454df800) 00000100:00000040:2.0:1775776704.861159:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861165:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776704.861167:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776704.861169:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720dd48. 00000100:00000200:2.0:1775776704.861172:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099784448, offset 0 00000400:00000200:2.0:1775776704.861176:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776704.861180:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776704.861183:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776704.861185:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776704.861188:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776704.861190:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776704.861192:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776704.861194:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4359:4359:256:4294967295] 192.168.201.139@tcp LPNI seq info [4359:4359:8:4294967295] 00000400:00000200:2.0:1775776704.861196:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776704.861200:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776704.861203:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776704.861205:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7a00. 00000800:00000200:2.0:1775776704.861208:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776704.861209:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776704.861212:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776704.861213:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1775776704.861216:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776704.861217:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:2.0:1775776704.861219:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776704.861221:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776704.861223:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dd48 00000400:00000010:2.0:1775776704.861224:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dd48. 00000100:00000001:2.0:1775776704.861225:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776704.861227:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861233:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.861236:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861253:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861255:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776704.861258:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776704.861261:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776704.861263:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776704.861265:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313092614 00000100:00000001:2.0:1775776704.861267:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313093877 00000100:00000001:2.0:1775776704.861269:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313092614 00000100:00000001:2.0:1775776704.861271:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313093877 00000100:00000001:2.0:1775776704.861273:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776704.861275:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861277:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776704.861281:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776704/real 1775776704] req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776704.861288:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776704.861290:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776704.861291:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776704.861294:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776704.861295:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776704.861296:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776704.861298:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776704.861303:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861307:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776704.861308:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861309:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.861309:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776704.861311:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861315:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776704.861317:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861321:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776704.861322:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861323:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.861324:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861325:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776704.861326:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776704.861328:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776704.861331:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776704.861333:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776704.861336:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776704.861343:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784448:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776704.861347:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776704.861350:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861356:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776704.861357:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.861360:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776704.861361:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776704.861363:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776704.861366:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784576:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776704.861369:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776704.861370:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776704.861372:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.861373:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776704.861376:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776704.861378:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.861381:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776704.861383:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776704.861385:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776704.861387:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099784576, portal 4 00000100:00000001:2.0:1775776704.861388:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776704.861389:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741759744 : -114841967791872 : ffff978d454df100) 00000100:00000040:2.0:1775776704.861393:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861397:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776704.861398:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776704.861399:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720dd48. 00000100:00000200:2.0:1775776704.861401:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099784576, offset 0 00000400:00000200:2.0:1775776704.861403:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776704.861405:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776704.861407:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776704.861409:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776704.861412:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776704.861414:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776704.861416:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776704.861419:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4360:4360:256:4294967295] 192.168.201.139@tcp LPNI seq info [4360:4360:8:4294967295] 00000400:00000200:2.0:1775776704.861423:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776704.861428:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776704.861432:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776704.861436:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7a00. 00000800:00000200:2.0:1775776704.861440:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776704.861443:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776704.861446:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776704.861449:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1775776704.861452:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776704.861454:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:2.0:1775776704.861457:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776704.861460:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776704.861462:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dd48 00000400:00000010:2.0:1775776704.861464:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dd48. 00000100:00000001:2.0:1775776704.861466:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776704.861468:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861474:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.861477:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861482:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861484:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776704.861487:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776704.861489:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776704.861492:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776704.861494:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313092623 00000100:00000001:2.0:1775776704.861496:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313093877 00000100:00000001:2.0:1775776704.861498:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313092623 00000100:00000001:2.0:1775776704.861500:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313093877 00000100:00000001:2.0:1775776704.861502:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776704.861505:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861506:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776704.861510:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776704/real 1775776704] req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776704.861517:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776704.861518:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776704.861520:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776704.861522:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776704.861524:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776704.861525:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776704.861528:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776704.861535:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861541:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776704.861542:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861544:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.861545:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776704.861548:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861554:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776704.861558:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861565:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776704.861567:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861569:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.861570:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861571:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776704.861573:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776704.861575:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776704.861578:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776704.861580:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776704.861582:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776704.861587:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784576:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776704.861589:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776704.861591:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861595:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776704.861596:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.861597:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776704.861598:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776704.861599:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776704.861601:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784704:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776704.861602:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776704.861603:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776704.861604:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776704.861605:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776704.861607:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776704.861609:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776704.861611:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776704.861612:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776704.861614:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776704.861616:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099784704, portal 4 00000100:00000001:2.0:1775776704.861617:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776704.861618:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741758848 : -114841967792768 : ffff978d454ded80) 00000100:00000040:2.0:1775776704.861621:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861625:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776704.861627:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776704.861628:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720dd48. 00000100:00000200:2.0:1775776704.861629:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099784704, offset 0 00000400:00000200:2.0:1775776704.861631:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776704.861634:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776704.861636:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776704.861638:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776704.861639:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776704.861641:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776704.861642:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776704.861644:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4361:4361:256:4294967295] 192.168.201.139@tcp LPNI seq info [4361:4361:8:4294967295] 00000400:00000200:2.0:1775776704.861647:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776704.861650:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776704.861653:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776704.861655:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7a00. 00000800:00000200:2.0:1775776704.861657:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776704.861659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1775776704.861661:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776704.861662:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1775776704.861665:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776704.861666:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:2.0:1775776704.861668:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776704.861671:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776704.861673:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dd48 00000400:00000010:2.0:1775776704.861675:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dd48. 00000100:00000001:2.0:1775776704.861677:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776704.861680:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861687:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.861690:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861695:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861697:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776704.861700:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776704.861703:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776704.861706:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776704.861707:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313092614 00000100:00000001:2.0:1775776704.861710:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313093877 00000100:00000001:2.0:1775776704.861711:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313092614 00000100:00000001:2.0:1775776704.861714:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313093877 00000100:00000001:2.0:1775776704.861715:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776704.861717:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861719:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776704.861722:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776704/real 1775776704] req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776704.861729:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776704.861730:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776704.861731:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776704.861733:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776704.861734:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776704.861735:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776704.861737:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776704.861743:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861749:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776704.861750:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861752:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.861753:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776704.861756:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861761:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776704.861764:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861771:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776704.861773:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861774:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776704.861775:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861795:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776704.861797:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776704.861799:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776704.861802:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776704.861804:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776704.861806:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776704.861812:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784704:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776704.861816:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861817:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.861820:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd500 x1862033099784320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861825:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776704.861827:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776704.861828:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776704.861831:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776704.861832:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776704.861833:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776704.861835:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776704.861836:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759ec00. 02000000:00000001:2.0:1775776704.861839:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776704.861841:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776704.861842:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd500. 00000100:00000001:2.0:1775776704.861844:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776704.861846:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861848:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.861851:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df800 x1862033099784448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861857:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776704.861858:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776704.861860:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776704.861862:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776704.861863:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776704.861864:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776704.861866:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776704.861867:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c000. 02000000:00000001:2.0:1775776704.861869:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776704.861870:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776704.861871:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df800. 00000100:00000001:2.0:1775776704.861873:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776704.861874:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861875:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.861878:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df100 x1862033099784576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861884:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776704.861885:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776704.861887:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776704.861889:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776704.861890:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776704.861891:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776704.861892:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776704.861893:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:2.0:1775776704.861895:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776704.861896:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776704.861898:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df100. 00000100:00000001:2.0:1775776704.861899:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776704.861900:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861901:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776704.861903:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ded80 x1862033099784704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776729 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776704.861913:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776704.861914:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776704.861915:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776704.861917:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776704.861918:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776704.861919:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776704.861920:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776704.861921:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775776704.861923:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776704.861924:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776704.861925:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776704.861926:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776704.861927:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861929:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776704.861932:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776704.861933:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776704.861935:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776704.861936:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776705.563871:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776705.563878:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776705.563880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776705.563887:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776705.563891:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776705.563895:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776705.563897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776705.563899:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776705.563902:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776705.627907:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776705.627910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776705.627915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776705.627916:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776705.627920:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776705.627923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776705.627925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776705.627926:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776705.627930:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776705.627931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776705.627933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776705.627934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776705.627936:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776705.627939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776705.627941:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776705.627946:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776705.627950:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776705.628011:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776705.628015:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776705.628018:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776706.139912:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313093877, at 4313093888 00000100:00000010:0.0:1775776706.139927:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000040:1.0:1775776706.140464:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313093877, at 4313093888 00000100:00000010:1.0:1775776706.140470:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:3.0:1775776706.587902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776706.587909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776706.587911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776706.587916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776706.587920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776706.587923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776706.587926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776706.587928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776706.587931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776706.651865:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1775776706.651866:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776706.651870:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776706.651872:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776706.651875:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776706.651876:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776706.651878:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776706.651881:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776706.651882:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776706.651883:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776706.651888:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776706.651888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776706.651890:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776706.651893:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776706.651896:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776706.651899:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776706.651902:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776706.651920:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776706.651925:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776706.651929:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776707.611893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776707.611899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776707.611900:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776707.611906:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776707.611909:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776707.611913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776707.611916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776707.611918:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776707.611921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776707.675935:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776707.675943:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776707.675945:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776707.675952:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776707.675954:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1775776707.675956:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776707.675963:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1775776707.675964:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776707.675969:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776707.675969:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1775776707.675973:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776707.675976:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776707.675977:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776707.675991:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776707.675994:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776707.675997:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776707.676000:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776707.676005:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776707.676007:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776707.676011:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776707.676012:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776707.676016:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776707.676017:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776707.676071:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776707.676074:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776707.676077:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776708.635890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776708.635896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776708.635898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776708.635903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776708.635906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776708.635909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776708.635911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776708.635913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776708.635915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776708.699914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776708.699919:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776708.699925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776708.699927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776708.699931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776708.699932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776708.699936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776708.699937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776708.702384:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776708.702389:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776708.702394:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776708.702396:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776708.702400:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776708.702402:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776708.702406:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776708.702410:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776708.702414:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776708.702417:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776708.702419:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776708.702422:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.659911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776709.659918:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776709.659920:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776709.659925:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776709.659928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776709.659932:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776709.659935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776709.659937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776709.659939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.723907:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776709.723911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776709.723915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.723915:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776709.723918:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776709.723922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776709.723924:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776709.723925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.723930:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776709.723931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776709.723934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.723934:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776709.723937:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776709.723938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776709.723940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.723941:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776709.723945:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776709.724004:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776709.724007:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776709.724009:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776709.979990:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776709.980001:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776709.980003:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776709.980007:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776709.980012:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776709.980030:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18425 00000400:00000200:3.0:1775776709.980046:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776709.980059:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776709.980062:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776709.980065:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.980075:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776709.980078:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.980081:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776709.980092:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776709.980094:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776709.980097:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776709.980105:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776709.980108:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776709.980111:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776709.980114:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.980116:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776709.980117:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.980119:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776709.980122:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776709.980134:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776709.980137:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776709.980170:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776709.980174:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776709.980177:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.980180:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776709.980182:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776709.980207:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776709.980211:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776709.980221:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776709.980223:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.980226:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776709.980229:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776709.980232:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776709.980240:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776709.980254:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776709.980270:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776709.980281:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776709.980286:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776709.980290:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776709.980324:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776709.980335:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776709.980339:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776709.980340:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776709.980342:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776709.980345:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776709.980354:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18425 00000400:00000200:3.0:1775776709.980358:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776709.980361:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775776709.980362:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776709.980366:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776709.980367:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776709.980371:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776709.980377:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000040:2.0:1775776709.980379:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776709.980380:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776709.980385:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776709.980394:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776709.980396:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776709.980397:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776709.980401:0:313512:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:2.0:1775776709.980402:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1775776709.980403:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776709.980406:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1775776709.980407:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776709.980408:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 02000000:00000001:2.0:1775776709.980413:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776709.980417:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099784832:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776709.980422:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776709.980425:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776709.980429:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.980432:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1775776709.980435:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 02000000:00000010:2.0:1775776709.980438:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 00000020:00000001:3.0:1775776709.980438:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 02000000:00000001:2.0:1775776709.980443:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.980450:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776709.980454:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776709.980460:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000040:3.0:1775776709.980464:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000200:2.0:1775776709.980466:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099784832, portal 25 00000100:00000001:2.0:1775776709.980469:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776709.980470:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751258880 : -114841958292736 : ffff978d45dee300) 00000100:00000001:3.0:1775776709.980476:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776709.980484:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776709.980487:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.980490:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776709.980499:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1775776709.980501:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000040:2.0:1775776709.980505:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00080000:3.0:1775776709.980507:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000010:2.0:1775776709.980509:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720dd48. 00000100:00000200:2.0:1775776709.980515:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099784832, offset 0 00000100:00000001:3.0:1775776709.980517:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776709.980520:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775776709.980524:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 02000000:00000001:3.0:1775776709.980530:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775776709.980535:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:3.0:1775776709.980535:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.980542:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:3.0:1775776709.980545:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775776709.980547:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776709.980552:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776709.980556:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000008:00100000:3.0:1775776709.980557:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1775776709.980559:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000008:00000001:3.0:1775776709.980562:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.980565:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4362:4362:256:4294967295] 192.168.201.139@tcp LPNI seq info [4362:4362:8:4294967295] 00000400:00000200:2.0:1775776709.980569:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776709.980572:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776709.980577:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000400:00000200:2.0:1775776709.980578:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776709.980583:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776709.980587:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7a00. 00000020:00000040:3.0:1775776709.980589:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 00000800:00000200:2.0:1775776709.980594:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:3.0:1775776709.980595:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776709.980598:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776709.980600:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776709.980612:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776709.980613:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775776709.980615:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.980619:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776709.980621:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313098997 02000000:00000001:3.0:1775776709.980622:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776709.980624:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc00. 02000000:00000010:3.0:1775776709.980628:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:2.0:1775776709.980631:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776709.980633:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.980638:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313098997 00000100:00000010:2.0:1775776709.980641:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc60. 00000100:00000001:2.0:1775776709.980645:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776709.980649:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1775776709.980649:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776709.980653:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.980657:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.980659:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776709.980668:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776709.980670:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775776709.980673:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776709.980675:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776709.980678:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.980683:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.980691:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776709.980701:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776709.980714:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776709.980749:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776709.980753:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776709.980760:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776709.980762:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776709.980764:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776709.980765:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776709.980767:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776709.980769:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1775776709.980772:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776709.980773:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776709.980814:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776709.980816:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.980819:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776709.980822:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776709.980825:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776709.980828:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18425 00000100:00100000:2.0:1775776709.980829:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784960:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776709.980837:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776709.980839:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776709.980841:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.980843:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1775776709.980844:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000010:2.0:1775776709.980847:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 00000400:00000200:3.0:1775776709.980850:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 02000000:00000001:2.0:1775776709.980851:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776709.980854:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775776709.980858:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775776709.980859:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.980863:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1775776709.980864:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601c948. 00000100:00000040:3.0:1775776709.980873:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.980875:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776709.980878:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776709.980885:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776709.980886:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776709.980888:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776709.980890:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776709.980892:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776709.980904:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000010:2.0:1775776709.980906:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000001:3.0:1775776709.980908:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1775776709.980912:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099784960, portal 10 00000100:00000040:3.0:1775776709.980914:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776709.980915:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776709.980917:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751255296 : -114841958296320 : ffff978d45ded500) 00000100:00000001:3.0:1775776709.980918:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776709.980923:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776709.980926:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000040:2.0:1775776709.980930:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776709.980931:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.980940:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1775776709.980943:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1775776709.980944:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776709.980948:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d7f8. 02000000:00000001:3.0:1775776709.980948:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1775776709.980953:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099784960, offset 0 00000400:00000200:2.0:1775776709.980958:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 02000000:00000001:3.0:1775776709.980959:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.980963:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775776709.980966:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000008:00100000:3.0:1775776709.980966:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1775776709.980972:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776709.980975:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776709.980977:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776709.980980:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000008:00000001:3.0:1775776709.980981:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.980985:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776709.980989:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4363:4363:255:4294967295] 192.168.201.139@tcp LPNI seq info [4363:4363:7:4294967295] 00000100:00000001:3.0:1775776709.980991:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.980995:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000010:3.0:1775776709.980998:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776709.981002:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000400:00000200:2.0:1775776709.981007:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776709.981011:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776709.981016:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7700. 02000000:00000001:3.0:1775776709.981016:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776709.981019:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776709.981023:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775776709.981027:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.1:1775776709.981027:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776709.981031:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776709.981031:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776709.981033:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776709.981036:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776709.981040:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776709.981042:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313097733 00000100:00000001:2.0:1775776709.981046:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313098997 02000000:00000010:3.0:1775776709.981046:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:2.0:1775776709.981048:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313097733 00000100:00000001:2.0:1775776709.981052:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313098997 00000100:00000001:3.0:1775776709.981052:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.981056:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776709.981057:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.981060:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.981062:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776709.981065:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776709.981068:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00100000:3.0:1775776709.981070:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776709.981071:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776709.981073:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776709.981075:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.981077:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.981088:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776709.981097:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776709.981101:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776709.981124:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776709.981126:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776709.981137:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776709.981139:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776709.981140:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776709.981143:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776709.981162:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18425 00000400:00000200:3.0:1775776709.981173:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776709.981177:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776709.981179:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776709.981181:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981183:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776709.981186:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981187:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776709.981190:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776709.981191:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776709.981192:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776709.981194:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776709.981196:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776709.981206:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776709.981209:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981210:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776709.981212:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981214:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776709.981217:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776709.981220:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981229:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776709.981231:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776709.981233:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776709.981235:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.981237:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776709.981239:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776709.981242:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.981257:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776709.981261:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776709.981263:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776709.981266:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776709.981274:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.981284:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776709.981286:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776709.981288:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776709.981292:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776709.981302:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776709.981314:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776709.981329:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776709.981331:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776709.981341:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776709.981356:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776709.981358:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776709.981368:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776709.981369:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776709.981371:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776709.981373:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776709.981383:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18425 00000400:00000200:3.0:1775776709.981388:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776709.981391:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776709.981402:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776709.981405:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981407:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776709.981416:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981418:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776709.981420:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776709.981422:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776709.981423:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776709.981425:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776709.981427:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776709.981429:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776709.981432:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981442:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776709.981451:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981453:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776709.981455:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776709.981458:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776709.981460:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776709.981462:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776709.981464:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776709.981466:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.981474:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776709.981476:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776709.981478:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.981480:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776709.981483:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776709.981485:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776709.981488:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776709.981495:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776709.981497:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776709.981499:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776709.981500:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776709.981504:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776709.981507:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776709.981511:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776709.981518:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776709.981520:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776709.981541:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776709.981558:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776709.981568:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18435) 00000100:00000001:2.0:1775776709.982880:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776709.982886:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776709.982888:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776709.982899:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776709.982909:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776709.982911:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.982915:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776709.982917:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776709.982920:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776709.982923:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785088:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776709.982926:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776709.982930:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776709.982932:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.982934:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776709.982942:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776709.982945:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.982952:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776709.982956:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776709.982964:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d2a8. 00000100:00000200:2.0:1775776709.982970:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099785088, portal 10 00000100:00000001:2.0:1775776709.982973:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776709.982974:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751260672 : -114841958290944 : ffff978d45deea00) 00000100:00000040:2.0:1775776709.982983:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776709.982990:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776709.982992:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776709.982995:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d440. 00000100:00000200:2.0:1775776709.982998:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099785088, offset 0 00000400:00000200:2.0:1775776709.983004:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776709.983012:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776709.983019:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776709.983023:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776709.983026:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776709.983029:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776709.983032:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776709.983035:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4364:4364:254:4294967295] 192.168.201.139@tcp LPNI seq info [4364:4364:6:4294967295] 00000400:00000200:2.0:1775776709.983040:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776709.983048:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776709.983054:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776709.983059:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7000. 00000800:00000200:2.0:1775776709.983065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776709.983069:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776709.983073:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776709.983082:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776709.983086:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776709.983088:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313097736 00000100:00000001:2.0:1775776709.983091:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313098997 00000100:00000001:2.0:1775776709.983093:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313097736 00000100:00000001:2.0:1775776709.983095:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313098997 00000100:00000001:2.0:1775776709.983097:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776709.983100:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.983103:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776709.983107:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776709.983113:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776709.983115:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.983117:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776709.983118:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776709.983120:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776709.983124:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785216:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776709.983127:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776709.983129:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776709.983130:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.983132:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776709.983136:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776709.983138:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.983141:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776709.983144:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ccb8. 00000400:00000010:2.0:1775776709.983158:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d220. 00000100:00000200:2.0:1775776709.983162:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099785216, portal 4 00000100:00000001:2.0:1775776709.983164:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776709.983166:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751254400 : -114841958297216 : ffff978d45ded180) 00000100:00000040:2.0:1775776709.983171:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776709.983178:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776709.983180:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776709.983182:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d990. 00000100:00000200:2.0:1775776709.983185:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099785216, offset 0 00000400:00000200:2.0:1775776709.983189:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776709.983193:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776709.983197:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776709.983200:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776709.983202:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776709.983204:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776709.983205:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776709.983207:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4365:4365:253:4294967295] 192.168.201.139@tcp LPNI seq info [4365:4365:5:4294967295] 00000400:00000200:2.0:1775776709.983211:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776709.983217:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776709.983221:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776709.983224:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776709.983228:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776709.983231:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776709.983234:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776709.983237:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776709.983240:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776709.983242:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313097745 00000100:00000001:2.0:1775776709.983243:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313098997 00000100:00000001:2.0:1775776709.983245:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313097745 00000100:00000001:2.0:1775776709.983247:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313098997 00000100:00000001:2.0:1775776709.983249:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776709.983251:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.983253:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776709.983257:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776709.983264:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776709.983265:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.983268:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776709.983269:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776709.983271:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776709.983274:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785344:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776709.983277:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776709.983279:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776709.983282:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776709.983284:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776709.983287:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776709.983289:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776709.983291:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776709.983294:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601c9a0. 00000400:00000010:2.0:1775776709.983297:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d110. 00000100:00000200:2.0:1775776709.983300:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099785344, portal 4 00000100:00000001:2.0:1775776709.983301:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776709.983303:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751249920 : -114841958301696 : ffff978d45dec000) 00000100:00000040:2.0:1775776709.983309:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776709.983315:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776709.983317:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776709.983320:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d198. 00000100:00000200:2.0:1775776709.983322:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099785344, offset 0 00000400:00000200:2.0:1775776709.983326:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776709.983330:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776709.983333:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776709.983335:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776709.983338:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776709.983339:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776709.983349:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776709.983351:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4366:4366:252:4294967295] 192.168.201.139@tcp LPNI seq info [4366:4366:4:4294967295] 00000400:00000200:2.0:1775776709.983354:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776709.983358:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776709.983361:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776709.983363:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7c00. 00000800:00000200:2.0:1775776709.983366:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776709.983368:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776709.983370:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776709.983371:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776709.983373:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776709.983375:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313097736 00000100:00000001:2.0:1775776709.983376:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313098997 00000100:00000001:2.0:1775776709.983378:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313097736 00000100:00000001:2.0:1775776709.983379:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313098997 00000100:00000001:2.0:1775776709.983380:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776709.983381:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.983383:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.983385:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776709.983390:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776709.983391:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775776709.983392:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776709.983393:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776709.983394:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776709.983396:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776710.683972:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776710.683980:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776710.683982:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776710.683987:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776710.683990:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776710.683994:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776710.683996:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776710.683998:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776710.684001:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776710.747878:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776710.747880:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776710.747884:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776710.747885:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776710.747888:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776710.747892:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776710.747897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776710.747898:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776710.747901:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776710.747903:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776710.747906:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776710.747910:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776710.747912:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776710.747913:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1775776710.747915:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776710.747916:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747920:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776710.747920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747924:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776710.747931:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776710.747933:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776710.747934:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747936:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776710.747938:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747940:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747942:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747946:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776710.747947:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776710.747948:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776710.747949:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747950:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776710.747951:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747952:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776710.747953:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776710.748007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776710.748010:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776710.748013:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776711.259907:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313098997, at 4313099008 00000100:00000010:0.0:1775776711.259923:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000040:1.0:1775776711.260014:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313098997, at 4313099008 00000100:00000010:1.0:1775776711.260021:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000001:3.0:1775776711.707881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776711.707888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776711.707890:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776711.707895:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776711.707899:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776711.707903:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776711.707905:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776711.707907:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776711.707910:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776711.771883:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776711.771887:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776711.771891:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776711.771891:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776711.771894:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776711.771899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776711.771900:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776711.771901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776711.771906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776711.771908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776711.771910:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776711.771911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776711.771915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776711.771916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776711.771917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776711.771918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776711.771923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776711.771975:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776711.771977:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776711.771980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776712.731910:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776712.731917:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776712.731919:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776712.731925:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776712.731928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776712.731932:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776712.731935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776712.731937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776712.731939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776712.795882:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776712.795883:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776712.795888:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776712.795889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776712.795893:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776712.795894:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776712.795897:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776712.795899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776712.795899:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776712.795901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776712.795904:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776712.795906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776712.795908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776712.795917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776712.795919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776712.795921:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776712.795923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776712.795967:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776712.795971:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776712.795974:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775776713.052055:0:301877:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775776713.052112:0:301877:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775776713.052122:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776713.052126:0:301877:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775776713.052131:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776713.052135:0:301877:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775776713.052141:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776713.052145:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7a00. 00000400:00000200:1.0:1775776713.052150:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776713.052155:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776713.052161:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dd48 00000400:00000010:1.0:1775776713.052163:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dd48. 00000100:00000001:1.0:1775776713.052168:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776713.052175:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052201:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776713.052206:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052213:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776713.052215:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776713.052223:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776713.052225:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7700. 00000400:00000200:1.0:1775776713.052228:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776713.052231:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776713.052233:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d7f8 00000400:00000010:1.0:1775776713.052235:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d7f8. 00000100:00000001:1.0:1775776713.052237:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776713.052241:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052248:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776713.052251:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052257:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776713.052259:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776713.052264:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776713.052266:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7000. 00000400:00000200:1.0:1775776713.052268:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776713.052271:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776713.052273:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d440 00000400:00000010:1.0:1775776713.052275:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d440. 00000100:00000001:1.0:1775776713.052277:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776713.052281:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052288:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776713.052291:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052297:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776713.052299:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776713.052303:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776713.052305:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:1.0:1775776713.052307:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776713.052310:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776713.052312:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d990 00000400:00000010:1.0:1775776713.052314:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d990. 00000100:00000001:1.0:1775776713.052315:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776713.052319:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052326:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776713.052328:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052335:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776713.052336:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776713.052340:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776713.052342:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7c00. 00000400:00000200:1.0:1775776713.052344:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776713.052347:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776713.052349:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d198 00000400:00000010:1.0:1775776713.052351:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d198. 00000100:00000001:1.0:1775776713.052353:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776713.052357:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052364:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776713.052366:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776734 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776713.052372:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776713.052373:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776713.052976:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776713.052981:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776713.052985:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776713.052994:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776709/real 1775776713] req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776713.053009:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776713.053010:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776713.053013:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776713.053018:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d2a8 00000400:00000010:2.0:1775776713.053019:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d2a8. 00000100:00000001:2.0:1775776713.053022:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776713.053025:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776713.053034:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053046:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776713.053047:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053051:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053052:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776713.053055:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053061:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776713.053065:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053073:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776713.053075:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053077:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053078:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053081:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776713.053083:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776713.053086:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776713.053091:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776713.053093:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776713.053108:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776713.053117:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785088:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776713.053122:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776713.053125:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776709/real 1775776713] req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776713.053133:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776713.053134:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776713.053137:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ccb8. 00000400:00000200:2.0:1775776713.053139:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d220 00000400:00000010:2.0:1775776713.053141:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d220. 00000100:00000001:2.0:1775776713.053142:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776713.053145:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776713.053152:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053157:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776713.053158:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053159:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053160:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776713.053163:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053169:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776713.053172:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053178:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776713.053181:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053182:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053183:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053185:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776713.053187:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776713.053189:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776713.053193:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776713.053195:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776713.053198:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776713.053205:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785216:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776713.053208:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776713.053211:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776709/real 1775776713] req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776713.053218:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776713.053219:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776713.053221:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601c9a0. 00000400:00000200:2.0:1775776713.053223:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d110 00000400:00000010:2.0:1775776713.053224:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d110. 00000100:00000001:2.0:1775776713.053226:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776713.053229:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776713.053236:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053242:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776713.053243:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053244:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053246:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776713.053249:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053256:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776713.053259:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053264:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776713.053266:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053267:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053269:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053270:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776713.053272:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776713.053274:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776713.053277:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776713.053279:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776713.053282:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776713.053289:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785344:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776713.053292:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776713.053295:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776709/real 1775776713] req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776713.053302:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776713.053303:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776713.053304:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601c948. 00000400:00000200:2.0:1775776713.053306:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776713.053308:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776713.053310:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776713.053313:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776713.053321:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053327:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776713.053328:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053330:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053331:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776713.053334:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053339:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776713.053343:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053355:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776713.053357:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053359:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053360:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053361:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776713.053363:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776713.053365:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776713.053368:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776713.053370:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776713.053374:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776713.053381:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099784960:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776713.053384:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776713.053387:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776709/real 1775776713] req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776713.053394:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776713.053395:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776713.053397:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776713.053399:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776713.053401:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776713.053403:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776713.053405:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776713.053413:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053419:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776713.053420:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053422:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053424:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776713.053427:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053433:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776713.053436:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053442:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776713.053444:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053446:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776713.053447:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053448:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776713.053450:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776713.053452:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776713.053455:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776713.053458:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776713.053462:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776713.053469:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099784832:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776713.053472:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053475:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776713.053478:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099785088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053485:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776713.053488:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776713.053491:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776713.053495:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776713.053498:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776713.053499:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776713.053502:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776713.053504:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:2.0:1775776713.053507:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776713.053508:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776713.053510:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:2.0:1775776713.053514:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776713.053516:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053517:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776713.053520:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099785216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053525:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776713.053526:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776713.053528:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776713.053530:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776713.053531:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776713.053532:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776713.053533:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776713.053535:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:2.0:1775776713.053537:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776713.053538:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776713.053539:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:2.0:1775776713.053541:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776713.053542:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053544:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776713.053547:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099785344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053553:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776713.053555:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776713.053556:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776713.053558:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776713.053559:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776713.053560:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776713.053562:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776713.053563:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:2.0:1775776713.053565:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776713.053566:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776713.053567:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:2.0:1775776713.053569:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776713.053570:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053572:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776713.053574:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded500 x1862033099784960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053580:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776713.053581:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776713.053582:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776713.053584:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776713.053585:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776713.053587:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776713.053588:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776713.053590:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:2.0:1775776713.053591:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776713.053592:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776713.053594:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded500. 00000100:00000001:2.0:1775776713.053596:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776713.053597:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053598:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776713.053601:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee300 x1862033099784832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776734 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776713.053608:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776713.053610:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776713.053611:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776713.053613:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776713.053614:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776713.053615:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776713.053617:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776713.053618:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:2.0:1775776713.053620:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776713.053621:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776713.053622:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee300. 00000100:00000001:2.0:1775776713.053624:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776713.053625:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053628:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776713.053635:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776713.053637:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776713.053639:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776713.053640:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776713.755901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776713.755908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776713.755910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776713.755915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776713.755919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776713.755922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776713.755925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776713.755928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776713.755930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776713.819945:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776713.819950:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776713.819953:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776713.819954:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776713.819956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776713.819957:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776713.819959:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776713.819960:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776713.820010:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776713.820016:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776713.820018:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776713.820022:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776713.820026:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775776713.820029:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000001:3.0:1775776713.820029:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776713.820033:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775776713.820034:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776713.820036:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1775776713.820039:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776713.820040:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776714.779901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776714.779908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776714.779910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776714.779914:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776714.779918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776714.779923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776714.779925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776714.779927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776714.779934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776714.843886:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776714.843891:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776714.843893:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776714.843895:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776714.843896:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776714.843903:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776714.843904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776714.843906:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776714.843909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776714.843911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776714.843913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776714.843914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776714.843919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776714.843921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776714.843922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776714.843923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776714.843926:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776714.843972:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776714.843975:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776714.843977:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776715.099899:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776715.099907:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776715.099909:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776715.099913:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776715.099916:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776715.099922:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18430 00000400:00000200:3.0:1775776715.099932:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776715.099937:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776715.099940:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776715.099943:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.099947:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.099949:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.099951:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776715.099954:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776715.099956:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776715.099958:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776715.099961:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776715.099963:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776715.099966:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776715.099969:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.099971:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.099973:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.099975:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776715.099978:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776715.099981:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776715.099983:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776715.099985:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776715.099988:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776715.099991:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.099994:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776715.099996:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776715.100002:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775776715.100005:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776715.100009:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776715.100011:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100014:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776715.100017:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776715.100021:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776715.100029:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776715.100035:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776715.100044:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776715.100055:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776715.100089:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776715.100095:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776715.100126:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776715.100129:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776715.100132:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776715.100134:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776715.100136:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776715.100138:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776715.100140:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18430 00000400:00000200:3.0:1775776715.100144:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776715.100147:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776715.100150:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776715.100153:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100155:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.100157:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100159:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776715.100161:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776715.100163:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776715.100164:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776715.100166:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776715.100168:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776715.100170:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776715.100173:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100175:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.100177:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100179:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776715.100181:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776715.100183:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100186:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776715.100187:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776715.100193:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776715.100195:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100197:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776715.100201:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776715.100204:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100206:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776715.100210:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776715.100212:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776715.100215:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776715.100216:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100218:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776715.100219:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776715.100220:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776715.100224:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776715.100227:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776715.100232:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776715.100240:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776715.100242:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776715.100250:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776715.100259:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776715.100261:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776715.100264:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776715.100265:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776715.100267:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776715.100269:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776715.100272:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18430 00000400:00000200:3.0:1775776715.100275:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776715.100278:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776715.100280:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776715.100283:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100285:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.100287:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100289:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776715.100291:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776715.100293:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776715.100295:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776715.100297:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776715.100299:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776715.100301:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776715.100304:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100305:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.100307:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100309:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776715.100311:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776715.100313:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100316:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776715.100317:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776715.100320:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776715.100322:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100324:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776715.100326:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776715.100328:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100330:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776715.100334:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776715.100336:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776715.100338:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776715.100339:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100341:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776715.100343:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776715.100344:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776715.100348:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775776715.100350:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776715.100355:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776715.100362:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776715.100364:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776715.100368:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776715.100375:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776715.100378:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776715.100381:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776715.100382:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776715.100384:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776715.100386:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776715.100388:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18430 00000400:00000200:3.0:1775776715.100392:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776715.100394:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776715.100396:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776715.100399:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100400:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.100402:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100404:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776715.100406:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776715.100407:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776715.100409:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776715.100411:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776715.100413:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776715.100415:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776715.100417:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100419:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.100420:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100422:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776715.100424:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776715.100426:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100428:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776715.100430:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776715.100432:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776715.100434:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100436:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776715.100438:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776715.100440:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100442:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776715.100445:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776715.100447:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776715.100450:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776715.100451:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100453:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776715.100454:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776715.100456:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776715.100459:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776715.100461:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776715.100465:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776715.100472:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776715.100473:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776715.100477:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776715.100484:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776715.100486:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776715.100489:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776715.100491:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776715.100492:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776715.100494:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776715.100496:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18430 00000400:00000200:3.0:1775776715.100499:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776715.100502:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776715.100504:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776715.100507:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100508:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.100510:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100511:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776715.100513:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776715.100514:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776715.100516:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776715.100518:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776715.100519:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776715.100521:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776715.100524:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100525:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776715.100527:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100528:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776715.100530:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776715.100532:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776715.100534:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776715.100536:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776715.100539:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776715.100541:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100543:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776715.100544:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776715.100547:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100549:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776715.100552:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776715.100554:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776715.100556:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776715.100557:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.100559:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776715.100561:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776715.100562:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776715.100566:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776715.100570:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776715.100574:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776715.100582:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776715.100584:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776715.100587:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776715.100595:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776715.100599:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18440) 00000100:00000001:2.0:1775776715.100891:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776715.100896:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776715.100899:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776715.100905:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.100913:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776715.100915:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.100918:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776715.100921:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776715.100924:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776715.100927:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099785472:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776715.100931:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776715.100936:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776715.100938:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.100941:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776715.100977:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776715.100979:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776715.100986:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776715.100990:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776715.101004:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776715.101013:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099785472, portal 25 00000100:00000001:2.0:1775776715.101015:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776715.101016:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751264256 : -114841958287360 : ffff978d45def800) 00000100:00000040:2.0:1775776715.101021:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101027:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776715.101030:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776715.101032:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776715.101036:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099785472, offset 0 00000400:00000200:2.0:1775776715.101043:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776715.101051:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.101056:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776715.101072:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776715.101076:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776715.101079:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776715.101082:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776715.101086:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4367:4367:256:4294967295] 192.168.201.139@tcp LPNI seq info [4367:4367:8:4294967295] 00000400:00000200:2.0:1775776715.101089:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.101100:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776715.101105:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.101111:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776715.101118:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776715.101121:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776715.101126:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776715.101129:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776715.101132:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.101135:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776715.101141:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776715.101146:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776715.101151:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776715.101153:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776715.101157:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776715.101164:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101174:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.101177:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101184:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.101186:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776715.101194:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776715.101198:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776715.101202:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313104118 00000100:00000010:2.0:1775776715.101208:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc60. 00000100:00000001:2.0:1775776715.101214:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776715.101221:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313104118 00000100:00000010:2.0:1775776715.101224:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc00. 00000100:00000001:2.0:1775776715.101228:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776715.101234:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776715.101236:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.101238:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776715.101243:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776715/real 1775776715] req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776715.101251:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776715.101253:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776715.101255:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776715.101257:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776715.101259:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776715.101261:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776715.101264:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776715.101276:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101282:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776715.101283:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101285:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.101288:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776715.101291:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101296:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776715.101301:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101307:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776715.101310:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101311:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.101312:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101315:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776715.101316:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776715.101323:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776715.101327:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776715.101329:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776715.101332:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776715.101339:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099785472:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776715.101344:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776715.101347:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101353:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776715.101355:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.101356:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776715.101358:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776715.101359:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776715.101362:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785600:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776715.101365:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776715.101367:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776715.101368:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.101369:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776715.101372:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776715.101374:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776715.101377:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776715.101379:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776715.101382:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776715.101384:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099785600, portal 10 00000100:00000001:2.0:1775776715.101386:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776715.101387:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257984 : -114841958293632 : ffff978d45dedf80) 00000100:00000040:2.0:1775776715.101392:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101397:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776715.101399:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776715.101401:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776715.101403:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099785600, offset 0 00000400:00000200:2.0:1775776715.101406:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776715.101416:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.101420:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776715.101422:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776715.101424:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776715.101426:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776715.101428:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776715.101431:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4368:4368:256:4294967295] 192.168.201.139@tcp LPNI seq info [4368:4368:8:4294967295] 00000400:00000200:2.0:1775776715.101434:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.101439:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776715.101443:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.101446:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776715.101450:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776715.101452:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776715.101455:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776715.101457:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776715.101460:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.101462:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776715.101464:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776715.101468:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776715.101470:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776715.101472:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776715.101474:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776715.101478:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101484:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.101487:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101492:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.101493:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776715.101496:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776715.101499:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776715.101502:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776715.101504:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313102853 00000100:00000001:2.0:1775776715.101506:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313104118 00000100:00000001:2.0:1775776715.101507:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313102853 00000100:00000001:2.0:1775776715.101509:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313104118 00000100:00000001:2.0:1775776715.101510:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776715.101512:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.101513:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776715.101517:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776715/real 1775776715] req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776715.101524:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776715.101525:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776715.101526:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776715.101528:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776715.101529:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776715.101531:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776715.101533:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776715.101540:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101545:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776715.101546:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101547:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.101548:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776715.101551:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101556:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776715.101560:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101565:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776715.101567:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101568:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.101569:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101570:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776715.101572:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776715.101574:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776715.101577:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776715.101579:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776715.101582:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776715.101588:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785600:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776715.101591:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776715.101594:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101599:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776715.101600:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.101602:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776715.101603:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776715.101605:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776715.101607:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785728:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776715.101609:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776715.101611:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776715.101612:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.101613:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776715.101616:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776715.101617:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776715.101620:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776715.101622:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776715.101624:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776715.101626:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099785728, portal 10 00000100:00000001:2.0:1775776715.101628:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776715.101629:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751258880 : -114841958292736 : ffff978d45dee300) 00000100:00000040:2.0:1775776715.101633:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101638:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776715.101640:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776715.101642:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776715.101644:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099785728, offset 0 00000400:00000200:2.0:1775776715.101648:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776715.101652:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.101654:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776715.101656:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776715.101658:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776715.101661:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776715.101663:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776715.101665:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4369:4369:256:4294967295] 192.168.201.139@tcp LPNI seq info [4369:4369:8:4294967295] 00000400:00000200:2.0:1775776715.101668:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.101673:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776715.101677:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.101680:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776715.101683:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776715.101685:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776715.101688:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776715.101690:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776715.101693:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.101694:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776715.101697:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776715.101699:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776715.101702:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776715.101703:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776715.101705:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776715.101708:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101714:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.101717:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101722:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.101723:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776715.101726:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776715.101728:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776715.101731:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776715.101732:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313102854 00000100:00000001:2.0:1775776715.101734:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313104118 00000100:00000001:2.0:1775776715.101736:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313102854 00000100:00000001:2.0:1775776715.101738:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313104118 00000100:00000001:2.0:1775776715.101740:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776715.101742:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.101744:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776715.101747:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776715/real 1775776715] req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776715.101754:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776715.101755:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776715.101756:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776715.101758:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776715.101760:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776715.101761:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776715.101764:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776715.101770:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101775:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776715.101776:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101815:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.101816:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776715.101820:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101826:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776715.101829:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101834:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776715.101837:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101838:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.101839:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.101841:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776715.101842:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776715.101844:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776715.101847:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776715.101849:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776715.101852:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776715.101859:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785728:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776715.101863:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776715.101867:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101872:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776715.101874:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.101876:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776715.101878:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776715.101879:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776715.101882:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785856:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776715.101885:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776715.101886:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776715.101888:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.101889:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776715.101893:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776715.101895:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776715.101898:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776715.101900:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776715.101903:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776715.101906:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099785856, portal 4 00000100:00000001:2.0:1775776715.101908:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776715.101909:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751255296 : -114841958296320 : ffff978d45ded500) 00000100:00000040:2.0:1775776715.101913:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.101919:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776715.101921:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776715.101923:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776715.101925:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099785856, offset 0 00000400:00000200:2.0:1775776715.101929:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776715.101933:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.101936:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776715.101938:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776715.101941:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776715.101943:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776715.101945:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776715.101947:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4370:4370:256:4294967295] 192.168.201.139@tcp LPNI seq info [4370:4370:8:4294967295] 00000400:00000200:2.0:1775776715.101951:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.101956:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776715.101959:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.101962:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776715.101966:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776715.101968:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776715.101971:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776715.101974:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776715.101977:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.101979:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776715.101981:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776715.101985:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776715.101988:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776715.101989:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776715.101991:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776715.101995:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102007:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.102009:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102015:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.102016:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776715.102019:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776715.102022:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776715.102024:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776715.102026:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313102864 00000100:00000001:2.0:1775776715.102028:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313104118 00000100:00000001:2.0:1775776715.102030:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313102864 00000100:00000001:2.0:1775776715.102032:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313104118 00000100:00000001:2.0:1775776715.102033:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776715.102036:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.102038:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776715.102041:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776715/real 1775776715] req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776715.102049:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776715.102050:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776715.102051:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776715.102054:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776715.102055:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776715.102057:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776715.102073:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776715.102080:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102085:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776715.102087:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102088:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.102089:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776715.102092:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102098:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776715.102101:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102106:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776715.102108:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102109:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.102111:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102112:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776715.102114:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776715.102116:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776715.102119:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776715.102122:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776715.102125:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776715.102131:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785856:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776715.102135:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776715.102138:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102143:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776715.102144:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.102145:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776715.102147:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776715.102148:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776715.102151:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785984:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776715.102153:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776715.102155:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776715.102156:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776715.102158:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776715.102161:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776715.102162:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776715.102164:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776715.102166:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776715.102169:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776715.102172:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099785984, portal 4 00000100:00000001:2.0:1775776715.102173:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776715.102174:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751249920 : -114841958301696 : ffff978d45dec000) 00000100:00000040:2.0:1775776715.102180:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102184:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776715.102186:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776715.102188:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776715.102190:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099785984, offset 0 00000400:00000200:2.0:1775776715.102193:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776715.102197:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.102200:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776715.102202:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776715.102204:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776715.102206:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776715.102208:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776715.102210:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4371:4371:256:4294967295] 192.168.201.139@tcp LPNI seq info [4371:4371:8:4294967295] 00000400:00000200:2.0:1775776715.102214:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776715.102219:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776715.102223:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.102226:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776715.102230:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776715.102232:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776715.102236:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776715.102238:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776715.102242:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776715.102244:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776715.102246:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776715.102249:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776715.102252:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776715.102253:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776715.102255:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776715.102259:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102266:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.102268:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776740 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102274:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.102275:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776715.102278:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776715.102281:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776715.102283:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776715.102285:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313102855 00000100:00000001:2.0:1775776715.102287:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313104118 00000100:00000001:2.0:1775776715.102289:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313102855 00000100:00000001:2.0:1775776715.102290:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313104118 00000100:00000001:2.0:1775776715.102292:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776715.102293:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.102295:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776715.102298:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776715/real 1775776715] req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776715.102304:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776715.102306:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776715.102308:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776715.102310:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776715.102311:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776715.102313:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776715.102316:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776715.102324:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102330:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776715.102331:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102333:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.102334:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776715.102338:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102343:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776715.102346:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102352:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776715.102354:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102356:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776715.102357:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102358:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776715.102360:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776715.102363:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776715.102365:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776715.102368:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776715.102371:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776715.102383:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099785984:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776715.102387:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102388:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.102391:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def800 x1862033099785472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102398:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776715.102400:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776715.102402:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776715.102405:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776715.102407:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776715.102408:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776715.102410:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776715.102412:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a000. 02000000:00000001:2.0:1775776715.102416:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776715.102418:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776715.102419:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def800. 00000100:00000001:2.0:1775776715.102423:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776715.102424:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102426:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.102429:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedf80 x1862033099785600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102435:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776715.102437:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776715.102438:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776715.102440:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776715.102441:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776715.102442:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776715.102444:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776715.102445:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115ac00. 02000000:00000001:2.0:1775776715.102447:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776715.102448:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776715.102449:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedf80. 00000100:00000001:2.0:1775776715.102452:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776715.102453:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102454:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.102457:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee300 x1862033099785728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102461:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776715.102462:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776715.102463:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776715.102465:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776715.102466:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776715.102467:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776715.102468:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776715.102469:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158000. 02000000:00000001:2.0:1775776715.102471:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776715.102472:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776715.102473:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee300. 00000100:00000001:2.0:1775776715.102475:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776715.102476:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102478:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.102480:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded500 x1862033099785856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102485:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776715.102486:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776715.102487:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776715.102489:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776715.102490:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776715.102490:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776715.102492:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776715.102493:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158800. 02000000:00000001:2.0:1775776715.102495:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776715.102496:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776715.102497:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded500. 00000100:00000001:2.0:1775776715.102498:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776715.102499:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102501:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776715.102504:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099785984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776740 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776715.102509:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776715.102510:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776715.102511:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776715.102512:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776715.102513:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776715.102514:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776715.102515:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776715.102517:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:2.0:1775776715.102518:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776715.102519:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776715.102520:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:2.0:1775776715.102522:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776715.102523:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102530:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776715.102537:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776715.102539:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776715.102541:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776715.102543:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.803872:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776715.803878:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776715.803880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776715.803886:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776715.803889:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776715.803892:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776715.803895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776715.803897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776715.803899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.867896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776715.867899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776715.867903:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.867903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776715.867907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776715.867908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776715.867911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776715.867913:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776715.867914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776715.867915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776715.867918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776715.867919:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776715.867925:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776715.867929:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776715.867932:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776715.867935:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776715.867938:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776715.867991:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776715.867995:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776715.867998:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776716.379881:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313104118, at 4313104128 00000100:00000010:1.0:1775776716.379896:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000040:0.0:1775776716.379906:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313104118, at 4313104128 00000100:00000010:0.0:1775776716.379914:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000001:3.0:1775776716.827896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776716.827904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776716.827906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776716.827910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776716.827913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776716.827916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776716.827918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776716.827920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776716.827922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776716.891907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776716.891913:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776716.891919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776716.891920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776716.891924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776716.891925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776716.891928:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776716.891929:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776716.891995:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776716.892000:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776716.892004:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776716.892247:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776716.892251:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776716.892254:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776716.892258:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776716.892263:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776716.892267:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776716.892269:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776716.892271:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776716.892274:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776717.851852:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776717.851859:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776717.851861:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776717.851867:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776717.851870:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776717.851873:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776717.851876:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776717.851878:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776717.851881:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776717.915879:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776717.915885:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776717.915888:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776717.915893:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776717.915897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776717.915901:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776717.915906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776717.915909:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776717.915912:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1775776717.915912:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776717.915917:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1775776717.915917:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776717.915923:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776717.915927:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776717.915929:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776717.915931:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776717.915934:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776717.915938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776717.915940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776717.915944:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776717.915945:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776717.915953:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776717.915954:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776717.916005:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776717.916008:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776717.916021:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776718.875937:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776718.875944:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776718.875946:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776718.875951:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776718.875954:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776718.875958:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776718.875960:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776718.875963:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776718.875965:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776718.939906:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776718.939910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776718.939913:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776718.939914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776718.939916:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776718.939922:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776718.939923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776718.939925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776718.939928:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776718.939930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776718.939932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776718.939932:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776718.939936:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776718.939938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776718.939939:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776718.939940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776718.939946:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776718.940018:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776718.940022:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776718.940025:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776719.899864:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776719.899869:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776719.899871:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776719.899875:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776719.899878:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776719.899881:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776719.899883:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776719.899885:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776719.899888:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776719.963892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776719.963897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776719.963901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776719.963902:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776719.963903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776719.963910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776719.963911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776719.963914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776719.963914:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776719.963918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776719.963920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776719.963921:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776719.963924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776719.963924:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776719.963930:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776719.963932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776719.963934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776719.964016:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776719.964020:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776719.964023:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776720.219917:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776720.219927:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776720.219930:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776720.219934:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776720.219938:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776720.219944:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18435 00000400:00000200:1.0:1775776720.219953:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776720.219958:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776720.219960:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776720.219962:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.219965:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776720.219967:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.219992:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776720.219997:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776720.219998:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776720.220001:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776720.220004:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776720.220006:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776720.220009:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776720.220012:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220014:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776720.220016:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220018:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776720.220021:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776720.220024:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220026:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776720.220029:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776720.220031:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776720.220034:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776720.220036:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776720.220038:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776720.220047:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776720.220051:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776720.220054:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776720.220056:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776720.220059:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776720.220061:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776720.220065:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776720.220073:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776720.220079:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776720.220088:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776720.220099:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776720.220102:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776720.220108:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776720.220124:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776720.220126:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776720.220128:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776720.220130:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776720.220131:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776720.220133:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776720.220134:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18435 00000400:00000200:1.0:1775776720.220137:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776720.220139:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776720.220140:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776720.220142:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220144:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776720.220146:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220154:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776720.220157:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776720.220158:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776720.220160:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776720.220162:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776720.220164:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776720.220166:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776720.220167:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1775776720.220170:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776720.220172:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776720.220173:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776720.220175:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776720.220177:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220180:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776720.220184:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000040:2.0:1775776720.220184:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776720.220191:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220198:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776720.220200:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776720.220200:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776720.220204:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776720.220206:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776720.220209:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.220209:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775776720.220212:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775776720.220213:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:1.0:1775776720.220217:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775776720.220217:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775776720.220223:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776720.220225:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099786112:192.168.201.139@tcp:250:kworker.0 00000100:00000001:1.0:1775776720.220227:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.220232:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1775776720.220234:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776720.220237:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776720.220238:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776720.220242:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776720.220242:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776720.220244:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776720.220247:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776720.220247:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775776720.220251:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776720.220253:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776720.220254:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000010:1.0:1775776720.220258:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 02000000:00000001:2.0:1775776720.220258:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776720.220264:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776720.220266:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776720.220270:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000100:00100000:1.0:1775776720.220273:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775776720.220277:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000001:1.0:1775776720.220282:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776720.220284:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776720.220286:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776720.220286:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099786112, portal 25 00000100:00000001:2.0:1775776720.220289:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776720.220292:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609984 : -114841421941632 : ffff978d65d6f480) 00000100:00000040:2.0:1775776720.220300:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776720.220302:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776720.220307:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776720.220308:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776720.220312:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00080000:1.0:1775776720.220314:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:2.0:1775776720.220315:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000001:1.0:1775776720.220316:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776720.220319:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1775776720.220319:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099786112, offset 0 00000100:00000001:1.0:1775776720.220324:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775776720.220327:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00080000:1.0:1775776720.220330:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18435 00000400:00000200:1.0:1775776720.220337:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776720.220340:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776720.220343:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775776720.220344:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:1.0:1775776720.220349:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220351:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776720.220354:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000040:1.0:1775776720.220355:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220358:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776720.220360:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:2.0:1775776720.220361:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:1.0:1775776720.220362:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776720.220364:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776720.220366:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776720.220368:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776720.220369:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000020:00000001:1.0:1775776720.220370:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776720.220374:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220377:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776720.220377:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1775776720.220381:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776720.220384:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220385:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776720.220388:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:2.0:1775776720.220392:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4372:4372:256:4294967295] 192.168.201.139@tcp LPNI seq info [4372:4372:8:4294967295] 00000100:00080000:1.0:1775776720.220396:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000200:2.0:1775776720.220400:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:1.0:1775776720.220401:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776720.220404:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776720.220407:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775776720.220411:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 02000000:00000001:1.0:1775776720.220412:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776720.220416:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776720.220420:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.0:1775776720.220420:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000008:00000001:1.0:1775776720.220424:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776720.220424:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000100:00000001:1.0:1775776720.220426:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776720.220432:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776720.220435:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000010:1.0:1775776720.220436:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776720.220439:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000200:2.1:1775776720.220441:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:1.0:1775776720.220443:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776720.220445:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776720.220446:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1775776720.220447:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776720.220450:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776720.220452:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775776720.220452:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.220455:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 02000000:00000010:1.0:1775776720.220458:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000400:00000200:2.0:1775776720.220460:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775776720.220462:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776720.220465:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776720.220470:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000100:00100000:1.0:1775776720.220471:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775776720.220472:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776720.220476:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775776720.220482:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1775776720.220483:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776720.220486:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776720.220495:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.220498:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.220504:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776720.220508:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776720.220513:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776720.220518:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776720.220519:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776720.220521:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776720.220522:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776720.220526:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776720.220528:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776720.220529:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776720.220532:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18435 00000400:00000200:2.0:1775776720.220537:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776720.220539:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776720.220541:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313109237 00000400:00000200:1.0:1775776720.220545:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000010:2.0:1775776720.220547:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bba0. 00000100:00080000:1.0:1775776720.220548:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776720.220553:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220556:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776720.220556:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000040:1.0:1775776720.220560:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220563:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776720.220565:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775776720.220565:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313109237 00000020:00000040:1.0:1775776720.220569:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000100:00000010:2.0:1775776720.220570:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bcc0. 00000020:00000001:1.0:1775776720.220571:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776720.220574:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000020:00000040:1.0:1775776720.220575:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776720.220579:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776720.220580:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775776720.220582:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776720.220584:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776720.220587:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776720.220588:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1775776720.220591:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1775776720.220595:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776720/real 1775776720] req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776720.220596:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220599:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776720.220604:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776720.220611:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220615:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775776720.220615:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1775776720.220619:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1775776720.220619:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776720.220622:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 02000000:00000001:1.0:1775776720.220624:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775776720.220626:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 02000000:00000001:1.0:1775776720.220628:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776720.220629:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:1.0:1775776720.220632:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776720.220634:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:1.0:1775776720.220635:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776720.220640:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776720.220640:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776720.220643:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776720.220649:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776720.220652:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00000200:2.0:1775776720.220653:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776720.220655:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776720.220657:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776720.220661:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776720.220663:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776720.220666:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.220666:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:1.0:1775776720.220669:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:2.0:1775776720.220669:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.220674:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776720.220675:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.220677:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776720.220681:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775776720.220686:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776720.220690:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776720.220693:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.220702:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776720.220708:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776720.220709:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.220725:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776720.220730:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.220733:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776720.220734:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.220734:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.220739:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1775776720.220740:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776720.220741:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776720.220747:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776720.220748:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776720.220751:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776720.220753:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775776720.220755:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:1.0:1775776720.220758:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776720.220759:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1775776720.220764:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18435 00000100:00000040:2.0:1775776720.220768:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776720.220771:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00100000:2.0:1775776720.220808:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099786112:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776720.220814:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.220817:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1775776720.220818:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000040:2.0:1775776720.220822:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f480 x1862033099786112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776720.220823:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776720.220832:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220837:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776720.220838:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000040:1.0:1775776720.220839:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000001:2.0:1775776720.220841:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1775776720.220842:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776720.220846:0:303560:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000010:2.0:1775776720.220846:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 00000020:00000040:1.0:1775776720.220847:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776720.220850:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776720.220852:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 02000000:00000001:2.0:1775776720.220852:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776720.220855:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776720.220855:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000001:1.0:1775776720.220858:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000040:2.0:1775776720.220858:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776720.220861:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000040:1.0:1775776720.220863:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000010:2.0:1775776720.220865:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776720.220866:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776720.220869:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776720.220870:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776720.220872:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 02000000:00000001:2.0:1775776720.220872:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776720.220875:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1775776720.220876:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000010:2.0:1775776720.220878:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f480. 00000100:00000001:1.0:1775776720.220879:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776720.220881:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776720.220884:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776720.220884:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1775776720.220887:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.220887:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776720.220892:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776720.220893:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1775776720.220894:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776720.220898:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776720.220899:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776720.220901:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776720.220902:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:2.0:1775776720.220903:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1775776720.220905:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775776720.220906:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776720.220907:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.220910:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776720.220911:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775776720.220913:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1775776720.220914:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775776720.220915:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1775776720.220917:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776720.220920:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775776720.220923:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:2.0:1775776720.220929:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776720.220930:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.220930:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.220934:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776720.220936:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776720.220939:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776720.220940:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775776720.220943:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786240:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776720.220948:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776720.220950:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776720.220952:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.220956:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775776720.220959:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:2.0:1775776720.220960:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 00000100:00000001:1.0:1775776720.220963:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776720.220968:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776720.220983:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776720.220988:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776720.220992:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776720.220997:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776720.221001:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099786240, portal 10 00000100:00000001:1.0:1775776720.221002:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221003:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776720.221005:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287596544 : -114841421955072 : ffff978d65d6c000) 00000100:00000040:1.0:1775776720.221010:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18445) 00000100:00000040:2.0:1775776720.221012:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221017:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776720.221019:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776720.221020:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776720.221022:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099786240, offset 0 00000400:00000200:2.0:1775776720.221025:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776720.221028:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776720.221031:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776720.221033:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776720.221035:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776720.221036:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776720.221038:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776720.221040:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4373:4373:256:4294967295] 192.168.201.139@tcp LPNI seq info [4373:4373:8:4294967295] 00000400:00000200:2.0:1775776720.221042:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776720.221046:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776720.221049:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.221051:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776720.221053:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776720.221055:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776720.221057:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776720.221059:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776720.221061:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.221062:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776720.221064:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776720.221067:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776720.221068:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776720.221069:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776720.221071:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776720.221072:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221077:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.221079:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221083:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221083:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776720.221085:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776720.221088:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776720.221090:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776720.221091:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313107973 00000100:00000001:2.0:1775776720.221093:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313109237 00000100:00000001:2.0:1775776720.221094:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313107973 00000100:00000001:2.0:1775776720.221095:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313109237 00000100:00000001:2.0:1775776720.221097:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776720.221098:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221099:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776720.221101:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776720/real 1775776720] req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776720.221105:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776720.221105:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776720.221106:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776720.221107:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776720.221108:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776720.221109:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776720.221111:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776720.221116:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221120:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776720.221120:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221121:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776720.221122:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776720.221124:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221127:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776720.221129:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221133:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776720.221134:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221135:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776720.221135:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221136:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776720.221138:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776720.221139:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776720.221141:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776720.221142:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776720.221144:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776720.221149:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786240:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776720.221151:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776720.221152:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221160:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776720.221161:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.221162:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776720.221163:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776720.221164:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776720.221166:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786368:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776720.221167:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776720.221168:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776720.221169:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.221170:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776720.221172:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776720.221173:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776720.221174:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776720.221175:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776720.221177:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776720.221178:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099786368, portal 10 00000100:00000001:2.0:1775776720.221179:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776720.221180:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287600128 : -114841421951488 : ffff978d65d6ce00) 00000100:00000040:2.0:1775776720.221183:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221187:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776720.221188:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776720.221189:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776720.221191:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099786368, offset 0 00000400:00000200:2.0:1775776720.221193:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776720.221195:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776720.221196:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776720.221198:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776720.221199:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776720.221200:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776720.221201:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776720.221203:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4374:4374:256:4294967295] 192.168.201.139@tcp LPNI seq info [4374:4374:8:4294967295] 00000400:00000200:2.0:1775776720.221206:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776720.221209:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776720.221212:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.221213:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776720.221215:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776720.221217:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776720.221220:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776720.221221:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776720.221223:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.221224:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776720.221226:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776720.221228:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776720.221229:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776720.221230:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776720.221231:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776720.221233:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221237:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.221239:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221242:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221243:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776720.221244:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776720.221246:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776720.221247:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776720.221248:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313107974 00000100:00000001:2.0:1775776720.221249:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313109237 00000100:00000001:2.0:1775776720.221250:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313107974 00000100:00000001:2.0:1775776720.221252:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313109237 00000100:00000001:2.0:1775776720.221253:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776720.221254:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221256:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776720.221258:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776720/real 1775776720] req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776720.221265:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776720.221267:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776720.221268:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776720.221270:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776720.221271:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776720.221273:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776720.221275:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776720.221283:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221289:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776720.221290:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221292:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776720.221293:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776720.221295:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221301:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776720.221304:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221310:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776720.221312:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221313:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776720.221314:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221316:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776720.221317:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776720.221319:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776720.221322:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776720.221324:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776720.221328:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776720.221334:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786368:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776720.221338:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776720.221340:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221346:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776720.221347:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.221349:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776720.221351:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776720.221352:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776720.221355:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786496:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776720.221358:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776720.221360:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776720.221361:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.221363:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776720.221366:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776720.221368:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776720.221371:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776720.221373:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776720.221376:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776720.221379:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099786496, portal 4 00000100:00000001:2.0:1775776720.221380:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776720.221382:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287608192 : -114841421943424 : ffff978d65d6ed80) 00000100:00000040:2.0:1775776720.221387:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221393:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776720.221396:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776720.221398:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776720.221400:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099786496, offset 0 00000400:00000200:2.0:1775776720.221404:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776720.221407:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776720.221411:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776720.221413:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776720.221415:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776720.221417:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776720.221420:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776720.221422:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4375:4375:256:4294967295] 192.168.201.139@tcp LPNI seq info [4375:4375:8:4294967295] 00000400:00000200:2.0:1775776720.221426:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776720.221432:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776720.221436:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.221439:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776720.221442:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776720.221444:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776720.221447:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776720.221449:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776720.221451:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.221452:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776720.221454:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776720.221456:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776720.221458:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776720.221459:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776720.221460:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776720.221463:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221467:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.221469:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221473:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221474:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776720.221476:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776720.221479:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776720.221481:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776720.221482:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313107983 00000100:00000001:2.0:1775776720.221483:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313109237 00000100:00000001:2.0:1775776720.221485:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313107983 00000100:00000001:2.0:1775776720.221486:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313109237 00000100:00000001:2.0:1775776720.221487:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776720.221488:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221489:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776720.221491:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776720/real 1775776720] req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776720.221495:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776720.221496:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776720.221497:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776720.221498:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776720.221499:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776720.221500:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776720.221502:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776720.221506:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221510:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776720.221510:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221511:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776720.221512:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776720.221514:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221517:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776720.221519:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221523:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776720.221525:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221526:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776720.221526:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221527:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776720.221528:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776720.221530:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776720.221531:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776720.221532:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776720.221534:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776720.221538:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786496:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776720.221540:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221541:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.221542:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c000 x1862033099786240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221545:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776720.221546:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776720.221547:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776720.221549:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776720.221549:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776720.221550:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776720.221551:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776720.221552:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38400. 02000000:00000001:2.0:1775776720.221554:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776720.221554:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776720.221555:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c000. 00000100:00000001:2.0:1775776720.221557:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776720.221558:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221559:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.221560:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ce00 x1862033099786368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221564:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776720.221565:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776720.221566:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776720.221567:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776720.221567:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776720.221568:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776720.221569:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776720.221569:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38c00. 02000000:00000001:2.0:1775776720.221570:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776720.221571:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776720.221572:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ce00. 00000100:00000001:2.0:1775776720.221573:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776720.221573:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221574:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.221576:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ed80 x1862033099786496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221579:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776720.221579:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776720.221580:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776720.221581:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776720.221582:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776720.221582:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776720.221583:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776720.221584:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39400. 02000000:00000001:2.0:1775776720.221585:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776720.221585:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776720.221586:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ed80. 00000100:00000001:2.0:1775776720.221587:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776720.221588:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221589:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221591:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776720.221592:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221593:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776720.221594:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776720.221595:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776720.221596:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221600:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776720.221601:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.221602:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776720.221603:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776720.221604:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776720.221606:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786624:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776720.221607:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776720.221609:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776720.221610:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776720.221611:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776720.221612:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776720.221613:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776720.221615:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776720.221616:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776720.221618:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776720.221620:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099786624, portal 4 00000100:00000001:2.0:1775776720.221621:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776720.221622:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601024 : -114841421950592 : ffff978d65d6d180) 00000100:00000040:2.0:1775776720.221625:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221628:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776720.221629:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776720.221631:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776720.221632:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099786624, offset 0 00000400:00000200:2.0:1775776720.221634:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776720.221636:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776720.221638:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776720.221640:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776720.221641:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776720.221643:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776720.221644:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776720.221646:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4376:4376:256:4294967295] 192.168.201.139@tcp LPNI seq info [4376:4376:8:4294967295] 00000400:00000200:2.0:1775776720.221648:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776720.221652:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776720.221654:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.221656:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776720.221658:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776720.221659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776720.221662:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776720.221663:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776720.221666:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776720.221668:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776720.221670:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776720.221673:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776720.221675:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776720.221677:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776720.221679:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776720.221682:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221688:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.221691:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776745 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221697:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221698:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776720.221701:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776720.221704:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776720.221707:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776720.221709:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313107974 00000100:00000001:2.0:1775776720.221711:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313109237 00000100:00000001:2.0:1775776720.221713:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313107974 00000100:00000001:2.0:1775776720.221715:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313109237 00000100:00000001:2.0:1775776720.221717:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776720.221719:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221720:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776720.221723:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776720/real 1775776720] req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776720.221729:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776720.221731:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776720.221732:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776720.221734:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776720.221736:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776720.221738:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776720.221740:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776720.221747:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221753:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776720.221755:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221756:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776720.221758:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776720.221761:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221767:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776720.221770:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221777:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776720.221797:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221798:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776720.221799:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221800:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776720.221801:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776720.221803:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776720.221805:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776720.221806:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776720.221809:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776720.221814:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786624:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776720.221816:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221817:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776720.221819:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d180 x1862033099786624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776745 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776720.221822:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776720.221823:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776720.221824:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776720.221825:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776720.221826:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776720.221827:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776720.221828:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776720.221829:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a400. 02000000:00000001:2.0:1775776720.221831:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776720.221831:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776720.221832:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d180. 00000100:00000001:2.0:1775776720.221834:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776720.221834:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221835:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776720.221837:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776720.221838:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776720.221839:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776720.221840:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776720.923870:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776720.923875:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776720.923876:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776720.923880:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776720.923882:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776720.923886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776720.923887:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776720.923889:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776720.923891:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776720.987875:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776720.987882:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776720.987886:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776720.987887:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776720.987890:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987892:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776720.987895:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776720.987896:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776720.987899:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1775776720.987900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776720.987903:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776720.987904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776720.987906:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1775776720.987907:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776720.987909:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776720.987910:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776720.987913:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776720.987914:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987917:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776720.987922:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776720.987923:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776720.987924:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987925:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776720.987925:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987926:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987928:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776720.987931:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776720.987932:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776720.987933:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987934:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776720.987935:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987936:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776720.987937:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776720.988001:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776720.988004:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776720.988006:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776721.499865:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313109237, at 4313109248 00000100:00000010:1.0:1775776721.499880:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000040:0.0:1775776721.499891:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313109237, at 4313109248 00000100:00000010:0.0:1775776721.499897:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000001:3.0:1775776721.947861:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776721.947867:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776721.947868:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776721.947872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776721.947874:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776721.947878:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776721.947879:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776721.947881:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776721.947883:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776722.011876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776722.011878:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776722.011881:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776722.011883:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776722.011883:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776722.011889:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776722.011891:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776722.011892:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776722.011894:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776722.011896:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776722.011899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776722.011899:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776722.011901:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776722.011902:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776722.011903:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776722.011907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776722.011909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776722.011976:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776722.011979:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776722.011985:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776722.971887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776722.971894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776722.971896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776722.971901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776722.971904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776722.971908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776722.971910:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776722.971913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776722.971915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776723.035889:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776723.035897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776723.035899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776723.035902:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776723.035904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776723.035906:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776723.035910:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776723.035912:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776723.035914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776723.035915:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776723.035919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776723.035919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776723.035934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776723.035938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776723.035939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776723.035967:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776723.035971:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776723.036008:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776723.036012:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776723.036015:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776723.995887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776723.995894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776723.995895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776723.995900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776723.995919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776723.995922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776723.995924:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776723.995930:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776723.995933:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776724.059887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776724.059892:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776724.059895:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776724.059896:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776724.059897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776724.059915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776724.059915:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776724.059918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776724.059920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776724.059923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776724.059924:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776724.059925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776724.059928:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776724.059930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776724.059931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776724.059933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776724.059935:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776724.060006:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776724.060009:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776724.060013:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776725.019917:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776725.019924:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776725.019926:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776725.019931:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776725.019934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776725.019938:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776725.019941:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776725.019943:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776725.019946:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776725.083897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776725.083903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776725.083904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776725.083904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776725.083908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776725.083910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776725.083913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776725.083915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776725.083916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776725.083918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776725.083919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776725.083922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776725.083924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776725.083925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776725.083928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776725.083933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776725.083934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776725.084002:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776725.084006:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776725.084009:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776725.339954:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776725.339962:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776725.339964:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776725.339967:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776725.339971:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776725.339976:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18440 00000400:00000200:1.0:1775776725.339984:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776725.339990:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776725.339993:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776725.339996:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340000:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776725.340003:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340005:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776725.340009:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776725.340011:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776725.340014:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776725.340017:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776725.340020:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776725.340022:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776725.340026:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340028:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776725.340030:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340031:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776725.340034:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776725.340037:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340039:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776725.340042:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776725.340044:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776725.340046:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340048:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776725.340050:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776725.340067:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776725.340071:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776725.340074:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776725.340076:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340079:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776725.340082:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776725.340085:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776725.340093:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776725.340099:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776725.340106:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776725.340120:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776725.340123:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776725.340127:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776725.340149:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776725.340152:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776725.340155:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776725.340156:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776725.340158:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776725.340161:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776725.340163:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18440 00000400:00000200:1.0:1775776725.340167:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776725.340170:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776725.340173:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776725.340176:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340178:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776725.340181:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340183:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776725.340185:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776725.340186:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776725.340187:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776725.340187:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1775776725.340189:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776725.340191:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776725.340192:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775776725.340193:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776725.340196:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776725.340197:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340200:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776725.340201:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340203:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776725.340205:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000040:2.0:1775776725.340206:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776725.340208:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340216:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776725.340220:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776725.340224:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1775776725.340226:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776725.340228:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776725.340229:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340233:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775776725.340237:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1775776725.340238:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776725.340243:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.340243:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776725.340246:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.340247:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776725.340251:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000100:00100000:2.0:1775776725.340253:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099786752:192.168.201.139@tcp:250:kworker.0 00000020:00000040:1.0:1775776725.340255:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776725.340261:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775776725.340261:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776725.340264:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340266:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776725.340267:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776725.340268:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776725.340271:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776725.340272:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.340276:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776725.340278:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776725.340283:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776725.340283:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776725.340287:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776725.340291:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776725.340297:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776725.340304:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000100:00000001:1.0:1775776725.340306:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776725.340309:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776725.340312:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000040:1.0:1775776725.340314:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776725.340320:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099786752, portal 25 00000100:00000001:2.0:1775776725.340325:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776725.340328:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.340328:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609088 : -114841421942528 : ffff978d65d6f100) 00000100:00080000:1.0:1775776725.340334:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776725.340338:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776725.340339:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776725.340343:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776725.340345:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776725.340351:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776725.340352:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1775776725.340356:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18440 00000100:00000040:2.0:1775776725.340360:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000200:1.0:1775776725.340364:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776725.340365:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000400:00000200:1.0:1775776725.340370:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776725.340373:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1775776725.340373:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099786752, offset 0 00000100:00000040:1.0:1775776725.340379:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340382:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776725.340386:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776725.340387:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340391:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776725.340394:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776725.340395:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776725.340396:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776725.340399:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000400:00000200:2.0:1775776725.340399:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:1.0:1775776725.340404:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776725.340407:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776725.340410:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000040:1.0:1775776725.340414:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340416:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776725.340418:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1775776725.340419:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340421:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776725.340423:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:2.0:1775776725.340424:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00080000:1.0:1775776725.340427:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340429:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776725.340431:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775776725.340432:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1775776725.340434:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776725.340438:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776725.340438:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340441:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776725.340443:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776725.340446:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776725.340447:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4377:4377:256:4294967295] 192.168.201.139@tcp LPNI seq info [4377:4377:8:4294967295] 00000100:00000001:1.0:1775776725.340449:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776725.340455:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000400:00000200:2.0:1775776725.340456:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000040:1.0:1775776725.340459:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776725.340462:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776725.340463:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340465:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776725.340467:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776725.340469:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776725.340472:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776725.340476:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776725.340482:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00100000:1.0:1775776725.340485:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1775776725.340493:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340500:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776725.340502:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776725.340503:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000100:00000040:1.0:1775776725.340508:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1775776725.340514:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:1.0:1775776725.340518:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776725.340520:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1775776725.340520:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000100:00080000:1.0:1775776725.340525:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776725.340527:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776725.340529:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776725.340532:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:2.1:1775776725.340532:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:1.0:1775776725.340537:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18440 00000800:00000200:2.1:1775776725.340537:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000400:00000200:1.0:1775776725.340542:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776725.340547:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000800:00000100:2.0:1775776725.340549:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00080000:1.0:1775776725.340550:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000800:00000010:2.0:1775776725.340552:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000100:00000040:1.0:1775776725.340556:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340559:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776725.340561:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776725.340561:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1775776725.340566:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776725.340569:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776725.340572:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000400:00000200:2.0:1775776725.340573:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000020:00000001:1.0:1775776725.340575:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776725.340577:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776725.340579:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776725.340581:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776725.340587:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340590:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776725.340590:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000100:00000040:1.0:1775776725.340594:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000010:2.0:1775776725.340594:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:1.0:1775776725.340596:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776725.340599:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776725.340601:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776725.340601:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775776725.340605:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776725.340608:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776725.340609:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776725.340612:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776725.340617:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340621:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776725.340625:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1775776725.340627:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000001:1.0:1775776725.340629:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776725.340633:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.340640:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.340642:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776725.340644:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.340649:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776725.340650:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776725.340654:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776725.340656:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776725.340657:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340658:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776725.340659:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776725.340661:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776725.340664:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000010:1.0:1775776725.340665:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:2.0:1775776725.340668:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313114357 00000100:00000001:1.0:1775776725.340670:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776725.340676:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:2.0:1775776725.340676:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775776725.340683:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776725.340689:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776725.340691:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313114357 00000100:00000001:1.0:1775776725.340692:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776725.340696:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b1e0. 00000100:00000040:1.0:1775776725.340699:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.340701:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776725.340707:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776725.340710:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.340712:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776725.340717:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776725/real 1775776725] req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776725.340720:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776725.340725:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776725.340733:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776725.340736:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776725.340737:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776725.340739:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776725.340742:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18440 00000400:00000001:2.0:1775776725.340746:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:1.0:1775776725.340747:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000001:2.0:1775776725.340748:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776725.340752:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:1.0:1775776725.340753:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776725.340757:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775776725.340758:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776725.340761:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000040:1.0:1775776725.340762:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776725.340766:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776725.340770:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776725.340797:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776725.340801:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340804:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776725.340806:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776725.340807:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776725.340809:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776725.340811:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776725.340814:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776725.340818:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000200:2.0:1775776725.340819:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776725.340824:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340828:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776725.340832:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340835:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776725.340836:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1775776725.340838:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00000001:2.0:1775776725.340840:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775776725.340841:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776725.340845:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.340845:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1775776725.340848:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776725.340848:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775776725.340853:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1775776725.340853:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776725.340858:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340862:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776725.340863:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1775776725.340866:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000040:2.0:1775776725.340868:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1775776725.340871:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776725.340876:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776725.340891:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776725.340894:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:2.0:1775776725.340896:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1775776725.340898:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775776725.340899:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775776725.340901:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.340902:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.340903:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776725.340906:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775776725.340906:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1775776725.340909:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775776725.340910:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1775776725.340912:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776725.340913:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000010:1.0:1775776725.340917:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00080000:2.0:1775776725.340918:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:1.0:1775776725.340922:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.340922:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776725.340929:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776725.340931:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776725.340954:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776725.340956:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776725.340960:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776725.340960:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099786752:192.168.201.139@tcp:250:kworker.0 00000100:00000001:1.0:1775776725.340971:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.340971:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.340976:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776725.340977:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18450) 00000100:00000040:2.0:1775776725.340982:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f100 x1862033099786752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.340990:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776725.340992:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776725.340995:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776725.340998:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776725.340999:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776725.341001:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776725.341003:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776725.341005:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3bc00. 02000000:00000001:2.0:1775776725.341008:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776725.341010:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776725.341011:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f100. 00000100:00000001:2.0:1775776725.341014:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776725.341015:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341017:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341024:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776725.341026:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341028:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776725.341029:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776725.341031:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776725.341034:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341041:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776725.341042:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.341043:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776725.341045:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776725.341046:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776725.341048:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786880:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776725.341056:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776725.341058:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776725.341059:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.341060:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776725.341062:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776725.341063:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776725.341066:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776725.341068:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776725.341071:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776725.341074:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099786880, portal 10 00000100:00000001:2.0:1775776725.341076:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776725.341077:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287598336 : -114841421953280 : ffff978d65d6c700) 00000100:00000040:2.0:1775776725.341083:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341089:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776725.341091:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776725.341093:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776725.341096:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099786880, offset 0 00000400:00000200:2.0:1775776725.341100:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776725.341104:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776725.341108:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776725.341110:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776725.341113:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776725.341115:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776725.341117:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776725.341120:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4378:4378:256:4294967295] 192.168.201.139@tcp LPNI seq info [4378:4378:8:4294967295] 00000400:00000200:2.0:1775776725.341124:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776725.341129:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776725.341133:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776725.341136:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776725.341140:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776725.341143:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776725.341147:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776725.341149:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776725.341153:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776725.341154:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776725.341157:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776725.341160:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776725.341164:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776725.341165:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776725.341168:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776725.341171:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341178:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776725.341181:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341193:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341194:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776725.341197:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776725.341201:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776725.341204:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776725.341206:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313113093 00000100:00000001:2.0:1775776725.341208:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313114357 00000100:00000001:2.0:1775776725.341210:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313113093 00000100:00000001:2.0:1775776725.341212:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313114357 00000100:00000001:2.0:1775776725.341214:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776725.341216:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341218:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776725.341221:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776725/real 1775776725] req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776725.341228:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776725.341229:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776725.341231:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776725.341232:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776725.341234:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776725.341236:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776725.341239:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776725.341247:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341253:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776725.341254:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341256:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.341257:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776725.341260:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341267:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776725.341271:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341278:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776725.341280:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341281:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.341282:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341283:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776725.341285:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776725.341287:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776725.341289:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776725.341291:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776725.341293:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776725.341298:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099786880:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776725.341300:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776725.341302:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341305:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776725.341306:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.341307:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776725.341308:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776725.341310:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776725.341311:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787008:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776725.341313:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776725.341314:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776725.341316:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.341317:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776725.341319:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776725.341320:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776725.341322:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776725.341323:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776725.341325:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776725.341327:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099787008, portal 10 00000100:00000001:2.0:1775776725.341328:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776725.341329:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287597440 : -114841421954176 : ffff978d65d6c380) 00000100:00000040:2.0:1775776725.341332:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341336:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776725.341337:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776725.341338:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776725.341340:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099787008, offset 0 00000400:00000200:2.0:1775776725.341342:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776725.341344:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776725.341347:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776725.341348:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776725.341350:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776725.341351:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776725.341353:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776725.341355:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4379:4379:256:4294967295] 192.168.201.139@tcp LPNI seq info [4379:4379:8:4294967295] 00000400:00000200:2.0:1775776725.341357:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776725.341361:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776725.341363:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776725.341365:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776725.341367:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776725.341369:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776725.341372:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776725.341373:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776725.341375:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776725.341377:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776725.341378:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776725.341380:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776725.341382:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776725.341383:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776725.341384:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776725.341386:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341390:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776725.341392:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341396:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341397:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776725.341398:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776725.341400:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776725.341401:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776725.341402:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313113094 00000100:00000001:2.0:1775776725.341404:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313114357 00000100:00000001:2.0:1775776725.341405:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313113094 00000100:00000001:2.0:1775776725.341406:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313114357 00000100:00000001:2.0:1775776725.341407:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776725.341408:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341409:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776725.341411:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776725/real 1775776725] req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776725.341415:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776725.341416:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776725.341417:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776725.341418:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776725.341419:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776725.341421:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776725.341422:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776725.341427:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341430:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776725.341431:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341432:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.341432:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776725.341434:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341437:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776725.341439:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341443:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776725.341444:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341445:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.341445:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341446:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776725.341447:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776725.341448:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776725.341450:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776725.341451:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776725.341453:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776725.341456:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787008:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776725.341459:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776725.341460:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341464:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776725.341465:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.341467:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776725.341467:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776725.341468:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776725.341470:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787136:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776725.341471:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776725.341472:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776725.341473:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.341474:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776725.341475:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776725.341476:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776725.341478:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776725.341479:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776725.341481:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776725.341483:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099787136, portal 4 00000100:00000001:2.0:1775776725.341484:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776725.341485:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287603712 : -114841421947904 : ffff978d65d6dc00) 00000100:00000040:2.0:1775776725.341488:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341491:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776725.341492:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776725.341494:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776725.341495:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099787136, offset 0 00000400:00000200:2.0:1775776725.341497:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776725.341498:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776725.341500:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776725.341502:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776725.341504:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776725.341506:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776725.341508:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776725.341511:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4380:4380:256:4294967295] 192.168.201.139@tcp LPNI seq info [4380:4380:8:4294967295] 00000400:00000200:2.0:1775776725.341514:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776725.341520:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776725.341524:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776725.341527:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776725.341530:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776725.341532:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776725.341536:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776725.341538:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776725.341542:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776725.341543:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776725.341546:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776725.341549:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776725.341552:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776725.341553:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776725.341556:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776725.341559:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341565:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776725.341567:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341571:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341571:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776725.341574:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776725.341575:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776725.341577:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776725.341579:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313113103 00000100:00000001:2.0:1775776725.341580:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313114357 00000100:00000001:2.0:1775776725.341581:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313113103 00000100:00000001:2.0:1775776725.341583:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313114357 00000100:00000001:2.0:1775776725.341584:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776725.341585:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341586:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776725.341588:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776725/real 1775776725] req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776725.341593:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776725.341594:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776725.341594:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776725.341596:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776725.341596:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776725.341598:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776725.341599:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776725.341604:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341607:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776725.341608:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341609:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.341609:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776725.341611:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341616:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776725.341619:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341625:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776725.341626:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341627:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.341628:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341630:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776725.341631:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776725.341633:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776725.341636:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776725.341638:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776725.341642:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776725.341650:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787136:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776725.341653:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776725.341656:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341662:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776725.341663:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.341665:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776725.341666:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776725.341667:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776725.341669:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787264:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776725.341671:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776725.341672:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776725.341673:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776725.341674:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776725.341676:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776725.341677:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776725.341679:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776725.341681:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776725.341682:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776725.341684:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099787264, portal 4 00000100:00000001:2.0:1775776725.341685:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776725.341686:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287611776 : -114841421939840 : ffff978d65d6fb80) 00000100:00000040:2.0:1775776725.341690:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341693:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776725.341695:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776725.341696:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776725.341698:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099787264, offset 0 00000400:00000200:2.0:1775776725.341700:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776725.341702:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776725.341704:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776725.341706:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776725.341707:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776725.341709:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776725.341711:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776725.341714:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4381:4381:256:4294967295] 192.168.201.139@tcp LPNI seq info [4381:4381:8:4294967295] 00000400:00000200:2.0:1775776725.341718:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776725.341723:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776725.341727:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776725.341730:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776725.341733:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776725.341735:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776725.341739:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776725.341741:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776725.341745:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776725.341746:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776725.341749:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776725.341752:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776725.341755:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776725.341757:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776725.341759:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776725.341762:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341767:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776725.341769:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776750 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341773:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341774:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776725.341776:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776725.341792:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776725.341794:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776725.341796:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313113095 00000100:00000001:2.0:1775776725.341797:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313114357 00000100:00000001:2.0:1775776725.341799:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313113095 00000100:00000001:2.0:1775776725.341800:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313114357 00000100:00000001:2.0:1775776725.341801:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776725.341802:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341803:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776725.341805:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776725/real 1775776725] req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776725.341810:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776725.341810:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776725.341811:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776725.341813:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776725.341814:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776725.341815:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776725.341817:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776725.341821:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341825:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776725.341826:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341827:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.341827:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776725.341829:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341832:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776725.341834:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341839:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776725.341840:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341841:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776725.341841:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341842:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776725.341843:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776725.341845:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776725.341846:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776725.341847:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776725.341849:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776725.341853:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787264:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776725.341855:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341856:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776725.341857:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c700 x1862033099786880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341861:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776725.341862:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776725.341863:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776725.341864:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776725.341865:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776725.341865:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776725.341867:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776725.341868:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39000. 02000000:00000001:2.0:1775776725.341869:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776725.341870:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776725.341871:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c700. 00000100:00000001:2.0:1775776725.341872:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776725.341873:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341874:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776725.341876:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c380 x1862033099787008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341892:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776725.341893:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776725.341894:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776725.341896:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776725.341897:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776725.341897:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776725.341898:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776725.341899:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b000. 02000000:00000001:2.0:1775776725.341901:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776725.341901:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776725.341902:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c380. 00000100:00000001:2.0:1775776725.341903:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776725.341904:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341904:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776725.341906:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6dc00 x1862033099787136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341910:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776725.341910:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776725.341911:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776725.341912:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776725.341913:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776725.341914:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776725.341915:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776725.341915:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a000. 02000000:00000001:2.0:1775776725.341916:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776725.341917:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776725.341917:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6dc00. 00000100:00000001:2.0:1775776725.341919:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776725.341919:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341920:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776725.341921:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6fb80 x1862033099787264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776750 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776725.341928:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776725.341929:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776725.341929:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776725.341930:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776725.341931:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776725.341932:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776725.341933:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776725.341934:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39c00. 02000000:00000001:2.0:1775776725.341936:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776725.341936:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776725.341938:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6fb80. 00000100:00000001:2.0:1775776725.341939:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776725.341940:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341942:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776725.341945:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776725.341946:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776725.341948:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776725.341949:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776726.043864:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776726.043894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776726.043895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776726.043900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776726.043902:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776726.043905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776726.043907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776726.043908:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776726.043910:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776726.107856:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776726.107861:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776726.107863:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776726.107864:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776726.107876:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776726.107881:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776726.107882:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776726.107886:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776726.107888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776726.107889:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776726.107890:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776726.107891:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776726.107895:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776726.107897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776726.107899:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776726.107903:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776726.107905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776726.107958:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776726.107961:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776726.107963:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776726.619882:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313114357, at 4313114368 00000100:00000010:1.0:1775776726.619895:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000040:0.0:1775776726.619981:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313114357, at 4313114368 00000100:00000010:0.0:1775776726.619988:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775776727.067897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776727.067904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776727.067905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776727.067909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776727.067913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776727.067917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776727.067919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776727.067921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776727.067924:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776727.131883:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776727.131888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776727.131889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776727.131892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776727.131893:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776727.131896:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776727.131898:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776727.131900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776727.131901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776727.131903:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776727.131906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776727.131906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776727.131909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776727.131911:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776727.131914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776727.131916:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776727.131917:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776727.131974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776727.131978:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776727.131980:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776728.091878:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776728.091883:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776728.091884:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776728.091888:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776728.091890:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776728.091893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776728.091895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776728.091897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776728.091898:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776728.155885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776728.155890:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776728.155890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776728.155892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776728.155896:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00000100:00100000:3.0:1775776728.155897:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776728.155900:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1775776728.155901:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776728.155904:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776728.155904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776728.155908:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776728.155909:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1775776728.155910:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776728.155912:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776728.155913:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776728.155916:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776728.155920:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776728.155923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776728.155925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776728.155928:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776728.155929:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776728.155933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776728.155934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776728.155988:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776728.155992:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776728.155996:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776729.115890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776729.115896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776729.115898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776729.115903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776729.115906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776729.115910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776729.115912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776729.115915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776729.115917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776729.179896:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776729.179896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776729.179903:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776729.179904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776729.179906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776729.179910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776729.179912:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776729.179912:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776729.179918:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776729.179921:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776729.179923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776729.179925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776729.179925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776729.179928:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776729.179930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776729.179932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776729.179933:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776729.179987:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776729.179990:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776729.179992:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776730.139923:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776730.139929:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776730.139931:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776730.139935:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776730.139937:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776730.139940:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776730.139942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776730.139944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776730.139946:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776730.203859:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776730.203864:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776730.203865:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776730.203868:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776730.203870:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776730.203873:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776730.203874:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776730.203877:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776730.203880:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776730.203880:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776730.203883:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776730.203884:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776730.203889:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776730.203891:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776730.203893:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776730.203897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776730.203898:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776730.203963:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776730.203967:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776730.203970:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776730.459896:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776730.459906:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776730.459908:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776730.459911:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776730.459916:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776730.459922:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18445 00000400:00000200:0.0:1775776730.459931:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776730.459938:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776730.459942:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776730.459945:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.459948:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776730.459951:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.459953:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776730.459957:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776730.459959:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776730.459961:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776730.459964:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776730.459965:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776730.459966:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776730.459968:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.459970:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776730.459971:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.459972:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776730.459974:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776730.459976:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776730.459977:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776730.459979:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776730.459982:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776730.459983:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.459985:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776730.459986:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776730.459994:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776730.459996:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776730.459999:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776730.460000:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460003:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776730.460005:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776730.460008:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776730.460013:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776730.460018:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776730.460024:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776730.460031:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776730.460034:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776730.460037:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776730.460055:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776730.460058:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776730.460060:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776730.460061:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776730.460062:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776730.460064:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776730.460065:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18445 00000400:00000200:0.0:1775776730.460068:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776730.460070:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776730.460072:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776730.460073:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460075:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776730.460076:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460078:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776730.460079:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776730.460080:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776730.460081:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776730.460082:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776730.460083:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776730.460084:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776730.460087:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460088:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776730.460090:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460092:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776730.460093:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776730.460095:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776730.460095:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776730.460096:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776730.460098:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776730.460099:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776730.460102:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1775776730.460103:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776730.460105:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460106:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:2.0:1775776730.460108:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775776730.460110:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776730.460115:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460118:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776730.460123:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000100:00000001:2.0:1775776730.460123:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1775776730.460126:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 00000100:00000001:2.0:1775776730.460127:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776730.460129:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776730.460131:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.460131:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775776730.460134:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776730.460134:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776730.460135:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776730.460137:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.460138:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776730.460140:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00100000:2.0:1775776730.460142:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099787392:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776730.460144:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460147:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1775776730.460150:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775776730.460150:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776730.460153:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.460156:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776730.460159:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776730.460163:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460168:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776730.460170:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776730.460173:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:0.0:1775776730.460176:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775776730.460177:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776730.460184:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000001:0.0:1775776730.460185:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776730.460188:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776730.460191:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776730.460192:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:2.0:1775776730.460192:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099787392, portal 25 00000100:00080000:0.0:1775776730.460195:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776730.460196:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776730.460198:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776730.460199:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741760640 : -114841967790976 : ffff978d454df480) 00000100:00080000:0.0:1775776730.460202:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18445 00000100:00000040:2.0:1775776730.460205:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776730.460210:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776730.460216:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:2.0:1775776730.460217:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1775776730.460220:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776730.460222:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776730.460225:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000040:0.0:1775776730.460226:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000200:2.0:1775776730.460228:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099787392, offset 0 00000100:00000001:0.0:1775776730.460229:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776730.460233:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776730.460236:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460237:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776730.460241:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776730.460242:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000400:00000200:2.0:1775776730.460243:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:0.0:1775776730.460246:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776730.460249:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000400:00000200:2.0:1775776730.460249:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000020:00000001:0.0:1775776730.460253:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776730.460256:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776730.460256:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776730.460261:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000040:0.0:1775776730.460263:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776730.460266:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775776730.460267:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776730.460269:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000040:0.0:1775776730.460272:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776730.460275:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4382:4382:256:4294967295] 192.168.201.139@tcp LPNI seq info [4382:4382:8:4294967295] 00000100:00000001:0.0:1775776730.460276:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776730.460282:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:2.0:1775776730.460282:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:0.0:1775776730.460287:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460293:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776730.460294:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 02000000:00000001:0.0:1775776730.460297:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1775776730.460300:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 02000000:00000001:0.0:1775776730.460301:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1775776730.460304:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 02000000:00000001:0.0:1775776730.460305:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460308:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776730.460310:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.0:1775776730.460313:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00000001:0.0:1775776730.460314:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460318:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776730.460319:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000010:0.0:1775776730.460321:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776730.460325:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000200:2.1:1775776730.460325:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776730.460328:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776730.460330:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776730.460330:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000100:00000001:0.0:1775776730.460333:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776730.460335:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776730.460337:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775776730.460337:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.460339:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 02000000:00000010:0.0:1775776730.460341:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776730.460345:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776730.460345:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:0.0:1775776730.460351:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776730.460357:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776730.460360:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776730.460361:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1775776730.460365:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776730.460367:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776730.460370:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:0.0:1775776730.460374:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776730.460376:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776730.460376:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1775776730.460378:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776730.460380:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776730.460381:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776730.460384:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:2.0:1775776730.460385:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776730.460387:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18445 00000400:00000200:0.0:1775776730.460394:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776730.460398:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776730.460402:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776730.460406:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460408:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776730.460409:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776730.460410:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460413:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776730.460416:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776730.460417:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000100:00000040:2.0:1775776730.460417:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776730.460420:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776730.460423:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776730.460427:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776730.460431:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776730.460433:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776730.460436:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776730.460440:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776730.460441:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776730.460444:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460447:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776730.460447:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776730.460451:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000400:00000200:2.0:1775776730.460455:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:0.0:1775776730.460458:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776730.460458:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313119477 00000100:00000001:0.0:1775776730.460463:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776730.460465:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b5a0. 02000000:00000001:0.0:1775776730.460466:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776730.460469:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776730.460471:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460473:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776730.460475:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776730.460477:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460477:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776730.460481:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776730.460484:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000100:00000001:2.0:1775776730.460484:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313119477 00000020:00000040:0.0:1775776730.460486:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776730.460489:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000010:2.0:1775776730.460489:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b7e0. 02000000:00000001:0.0:1775776730.460490:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460493:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776730.460494:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775776730.460494:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1775776730.460497:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460498:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1775776730.460501:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:2.0:1775776730.460501:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460505:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775776730.460506:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776730.460510:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776730/real 1775776730] req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776730.460512:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1775776730.460528:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776730.460530:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776730.460532:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776730.460534:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776730.460535:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776730.460536:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776730.460540:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776730.460553:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776730.460557:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776730.460560:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776730.460564:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.460581:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776730.460582:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460583:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776730.460586:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776730.460587:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.460590:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776730.460592:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776730.460594:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776730.460596:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775776730.460597:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776730.460601:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776730.460607:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18445 00000100:00000001:2.0:1775776730.460613:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776730.460614:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:2.0:1775776730.460618:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776730.460619:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776730.460623:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776730.460628:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460632:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776730.460633:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1775776730.460635:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776730.460637:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776730.460638:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776730.460639:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1775776730.460640:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775776730.460641:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775776730.460642:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776730.460644:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776730.460644:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000040:0.0:1775776730.460647:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000100:00000001:2.0:1775776730.460648:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1775776730.460650:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00080000:2.0:1775776730.460651:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:0.0:1775776730.460653:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776730.460657:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00080000:2.0:1775776730.460657:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776730.460662:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776730.460662:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776730.460664:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460667:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000040:2.0:1775776730.460668:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776730.460671:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776730.460676:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776730.460680:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776730.460683:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775776730.460683:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099787392:192.168.201.139@tcp:250:kworker.0 02000000:00000001:0.0:1775776730.460687:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776730.460689:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460690:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776730.460690:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1775776730.460692:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1775776730.460694:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00000001:0.0:1775776730.460695:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460697:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776730.460700:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000100:00000040:2.0:1775776730.460700:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df480 x1862033099787392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776730.460703:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776730.460706:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776730.460708:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460712:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776730.460714:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775776730.460716:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776730.460717:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.460719:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776730.460722:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 02000000:00000010:2.0:1775776730.460722:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 00000100:00000001:0.0:1775776730.460727:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.460728:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776730.460731:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776730.460733:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776730.460735:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:0.0:1775776730.460736:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1775776730.460737:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:2.0:1775776730.460742:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776730.460745:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776730.460747:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df480. 00000100:00000001:0.0:1775776730.460751:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776730.460751:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776730.460752:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.460754:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776730.460756:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460760:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776730.460761:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460763:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776730.460764:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776730.460766:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776730.460768:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776730.460772:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.460825:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776730.460827:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776730.460829:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.460829:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776730.460832:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776730.460834:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776730.460836:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18455) 00000100:00100000:2.0:1775776730.460836:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787520:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776730.460841:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776730.460843:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776730.460844:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.460845:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776730.460847:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776730.460849:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776730.460851:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776730.460853:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776730.460855:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776730.460857:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099787520, portal 10 00000100:00000001:2.0:1775776730.460859:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776730.460859:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741753472 : -114841967798144 : ffff978d454dd880) 00000100:00000040:2.0:1775776730.460864:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.460868:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776730.460870:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776730.460872:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776730.460874:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099787520, offset 0 00000400:00000200:2.0:1775776730.460880:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776730.460883:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776730.460885:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776730.460887:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776730.460889:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776730.460890:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776730.460892:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776730.460894:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4383:4383:256:4294967295] 192.168.201.139@tcp LPNI seq info [4383:4383:8:4294967295] 00000400:00000200:2.0:1775776730.460896:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776730.460900:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776730.460903:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.460905:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776730.460908:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776730.460909:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776730.460912:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776730.460913:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776730.460915:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.460917:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776730.460919:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776730.460921:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776730.460923:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776730.460924:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776730.460925:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776730.460927:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.460932:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776730.460934:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.460938:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460939:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776730.460940:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776730.460942:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776730.460944:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776730.460945:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313118213 00000100:00000001:2.0:1775776730.460947:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313119477 00000100:00000001:2.0:1775776730.460948:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313118213 00000100:00000001:2.0:1775776730.460949:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313119477 00000100:00000001:2.0:1775776730.460950:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776730.460951:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.460953:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776730.460954:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776730/real 1775776730] req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776730.460959:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776730.460960:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776730.460961:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776730.460962:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776730.460963:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776730.460964:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776730.460966:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776730.460970:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.460974:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776730.460975:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.460976:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.460976:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776730.460978:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.460981:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776730.460983:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.460987:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776730.460988:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.460989:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.460990:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.460991:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776730.460992:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776730.460993:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776730.460995:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776730.460996:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776730.460998:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776730.461002:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787520:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776730.461004:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776730.461006:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461013:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776730.461014:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.461015:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776730.461016:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776730.461017:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776730.461019:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787648:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776730.461020:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776730.461021:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776730.461022:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.461023:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776730.461025:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776730.461026:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776730.461028:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776730.461029:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776730.461030:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776730.461032:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099787648, portal 10 00000100:00000001:2.0:1775776730.461033:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776730.461034:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748992 : -114841967802624 : ffff978d454dc700) 00000100:00000040:2.0:1775776730.461036:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461040:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776730.461042:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776730.461043:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776730.461044:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099787648, offset 0 00000400:00000200:2.0:1775776730.461046:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776730.461048:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776730.461049:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776730.461051:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776730.461052:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776730.461053:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776730.461054:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776730.461056:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4384:4384:256:4294967295] 192.168.201.139@tcp LPNI seq info [4384:4384:8:4294967295] 00000400:00000200:2.0:1775776730.461059:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776730.461063:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776730.461065:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.461067:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776730.461069:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776730.461071:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776730.461073:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776730.461074:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776730.461076:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.461077:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776730.461079:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776730.461081:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776730.461083:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776730.461084:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776730.461085:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776730.461087:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461091:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776730.461092:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461096:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.461097:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776730.461098:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776730.461099:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776730.461101:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776730.461103:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313118214 00000100:00000001:2.0:1775776730.461104:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313119477 00000100:00000001:2.0:1775776730.461105:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313118214 00000100:00000001:2.0:1775776730.461106:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313119477 00000100:00000001:2.0:1775776730.461107:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776730.461109:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.461110:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776730.461111:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776730/real 1775776730] req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776730.461116:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776730.461116:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776730.461117:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776730.461118:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776730.461119:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776730.461120:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776730.461122:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776730.461126:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461130:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776730.461130:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461131:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.461132:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776730.461134:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461137:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776730.461139:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461142:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776730.461143:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461144:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.461145:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461146:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776730.461146:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776730.461148:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776730.461150:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776730.461151:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776730.461153:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776730.461157:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787648:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776730.461159:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776730.461161:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461164:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776730.461165:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.461166:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776730.461167:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776730.461168:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776730.461169:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787776:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776730.461171:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776730.461172:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776730.461173:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.461174:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776730.461175:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776730.461176:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776730.461178:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776730.461179:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776730.461181:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776730.461182:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099787776, portal 4 00000100:00000001:2.0:1775776730.461183:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776730.461184:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741750784 : -114841967800832 : ffff978d454dce00) 00000100:00000040:2.0:1775776730.461187:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461191:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776730.461192:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776730.461193:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776730.461194:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099787776, offset 0 00000400:00000200:2.0:1775776730.461196:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776730.461198:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776730.461200:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776730.461202:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776730.461203:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776730.461205:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776730.461206:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776730.461207:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4385:4385:256:4294967295] 192.168.201.139@tcp LPNI seq info [4385:4385:8:4294967295] 00000400:00000200:2.0:1775776730.461210:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776730.461213:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776730.461216:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.461217:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776730.461219:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776730.461221:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776730.461223:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776730.461224:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776730.461226:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.461228:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776730.461229:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776730.461231:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776730.461232:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776730.461233:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776730.461234:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776730.461236:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461240:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776730.461241:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461245:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.461246:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776730.461247:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776730.461249:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776730.461251:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776730.461252:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313118223 00000100:00000001:2.0:1775776730.461253:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313119477 00000100:00000001:2.0:1775776730.461254:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313118223 00000100:00000001:2.0:1775776730.461256:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313119477 00000100:00000001:2.0:1775776730.461257:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776730.461258:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.461259:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776730.461260:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776730/real 1775776730] req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776730.461264:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776730.461265:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776730.461265:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776730.461266:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776730.461267:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776730.461268:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776730.461269:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776730.461273:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461277:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776730.461278:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461279:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.461280:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776730.461282:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461285:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776730.461287:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461291:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776730.461293:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461293:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.461294:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461295:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776730.461296:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776730.461297:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776730.461299:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776730.461300:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776730.461301:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776730.461305:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787776:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776730.461307:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461308:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776730.461310:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd880 x1862033099787520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461313:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776730.461314:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776730.461315:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776730.461316:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776730.461317:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776730.461317:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776730.461318:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776730.461319:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:2.0:1775776730.461320:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776730.461321:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776730.461322:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd880. 00000100:00000001:2.0:1775776730.461324:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776730.461324:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461326:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776730.461327:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc700 x1862033099787648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461331:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776730.461331:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776730.461332:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776730.461333:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776730.461334:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776730.461334:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776730.461335:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776730.461336:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:2.0:1775776730.461337:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776730.461338:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776730.461338:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc700. 00000100:00000001:2.0:1775776730.461339:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776730.461340:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461341:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776730.461342:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dce00 x1862033099787776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461345:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776730.461346:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776730.461346:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776730.461347:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776730.461348:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776730.461349:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776730.461349:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776730.461350:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:2.0:1775776730.461351:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776730.461352:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776730.461352:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dce00. 00000100:00000001:2.0:1775776730.461353:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776730.461354:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461355:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461357:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776730.461358:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.461359:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776730.461360:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776730.461360:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776730.461362:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461366:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776730.461367:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.461368:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776730.461369:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776730.461370:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776730.461372:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787904:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776730.461374:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776730.461376:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776730.461377:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776730.461379:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776730.461381:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776730.461383:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776730.461386:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776730.461388:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776730.461390:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776730.461393:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099787904, portal 4 00000100:00000001:2.0:1775776730.461394:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776730.461395:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748096 : -114841967803520 : ffff978d454dc380) 00000100:00000040:2.0:1775776730.461400:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461407:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776730.461409:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776730.461411:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776730.461413:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099787904, offset 0 00000400:00000200:2.0:1775776730.461417:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776730.461420:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776730.461423:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776730.461426:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776730.461428:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776730.461430:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776730.461432:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776730.461434:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4386:4386:256:4294967295] 192.168.201.139@tcp LPNI seq info [4386:4386:8:4294967295] 00000400:00000200:2.0:1775776730.461436:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776730.461440:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776730.461443:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.461445:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776730.461447:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776730.461449:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776730.461451:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776730.461452:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776730.461454:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776730.461456:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776730.461457:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776730.461459:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776730.461461:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776730.461462:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776730.461463:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776730.461465:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461470:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776730.461471:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776755 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461475:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.461476:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776730.461477:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776730.461479:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776730.461481:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776730.461482:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313118214 00000100:00000001:2.0:1775776730.461484:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313119477 00000100:00000001:2.0:1775776730.461485:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313118214 00000100:00000001:2.0:1775776730.461486:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313119477 00000100:00000001:2.0:1775776730.461488:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776730.461489:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.461490:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776730.461492:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776730/real 1775776730] req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776730.461496:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776730.461497:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776730.461497:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776730.461498:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776730.461499:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776730.461500:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776730.461502:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776730.461507:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461510:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776730.461511:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461511:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.461512:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776730.461515:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461521:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776730.461524:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461529:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776730.461532:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461533:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776730.461534:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461535:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776730.461537:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776730.461540:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776730.461543:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776730.461545:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776730.461549:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776730.461557:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099787904:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776730.461560:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461562:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776730.461565:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc380 x1862033099787904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776755 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776730.461571:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776730.461572:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776730.461573:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776730.461575:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776730.461576:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776730.461578:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776730.461579:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776730.461581:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d000. 02000000:00000001:2.0:1775776730.461583:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776730.461584:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776730.461585:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc380. 00000100:00000001:2.0:1775776730.461587:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776730.461589:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461590:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776730.461593:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776730.461595:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776730.461596:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776730.461598:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776731.163865:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776731.163871:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776731.163872:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776731.163876:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776731.163879:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776731.163882:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776731.163885:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776731.163887:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776731.163889:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776731.227878:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776731.227883:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776731.227883:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776731.227884:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227887:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776731.227889:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776731.227893:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776731.227893:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776731.227896:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776731.227898:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1775776731.227899:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776731.227900:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776731.227903:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1775776731.227903:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227908:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227911:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776731.227914:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227916:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227919:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776731.227925:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776731.227926:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776731.227927:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227929:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776731.227930:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227931:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227933:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776731.227937:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776731.227938:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776731.227940:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227941:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776731.227942:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227944:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776731.227945:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776731.227997:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776731.228000:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776731.228003:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776731.739900:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313119477, at 4313119488 00000100:00000010:0.0:1775776731.739915:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000040:1.0:1775776731.739950:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313119477, at 4313119488 00000100:00000010:1.0:1775776731.739962:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:3.0:1775776732.187887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776732.187894:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776732.187896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776732.187901:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776732.187905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776732.187909:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776732.187912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776732.187914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776732.187916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776732.251864:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776732.251870:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776732.251875:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776732.251876:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776732.251877:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776732.251881:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776732.251882:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776732.251883:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776732.251887:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776732.251887:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776732.251889:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776732.251894:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776732.251900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776732.251904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776732.251908:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776732.251911:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776732.251915:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776732.251971:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776732.251975:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776732.251977:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776733.211928:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776733.211936:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776733.211938:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776733.211944:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776733.211948:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776733.211953:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776733.211955:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776733.211958:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776733.211961:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776733.275903:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776733.275907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776733.275911:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776733.275912:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776733.275913:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776733.275919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776733.275919:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776733.275920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776733.275923:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776733.275924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776733.275925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776733.275926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776733.275928:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776733.275930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776733.275930:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776733.275932:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776733.275935:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776733.276008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776733.276012:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776733.276016:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776734.235932:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776734.235941:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776734.235944:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776734.235951:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776734.235956:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776734.235961:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776734.235965:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776734.235968:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776734.235971:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776734.299932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776734.299939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776734.299947:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776734.299948:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776734.299952:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776734.299953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776734.299958:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776734.299959:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776734.300029:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776734.300033:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776734.300037:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776734.300812:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776734.300817:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776734.300820:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776734.300825:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776734.300830:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776734.300834:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776734.300837:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776734.300840:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776734.300843:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776735.259907:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776735.259914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776735.259916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776735.259922:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776735.259925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776735.259929:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776735.259932:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776735.259934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776735.259936:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776735.323944:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776735.323951:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776735.323958:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776735.323960:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776735.323963:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776735.323965:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776735.323968:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776735.323969:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776735.324040:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776735.324045:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776735.324048:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776735.324100:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776735.324105:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776735.324107:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776735.324111:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776735.324116:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776735.324120:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776735.324123:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776735.324125:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776735.324128:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776735.579942:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776735.579950:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776735.579952:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776735.579955:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776735.579960:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776735.579971:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18450 00000400:00000200:0.0:1775776735.579981:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776735.579987:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776735.579990:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776735.579994:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.579997:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776735.579999:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580002:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776735.580006:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776735.580008:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776735.580011:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776735.580014:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776735.580016:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776735.580018:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776735.580021:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580023:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776735.580025:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580027:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776735.580029:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776735.580032:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580035:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776735.580038:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776735.580042:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776735.580045:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580048:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776735.580050:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776735.580060:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776735.580064:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776735.580077:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776735.580078:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580081:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776735.580083:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776735.580086:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776735.580094:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776735.580099:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776735.580106:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776735.580114:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776735.580118:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776735.580123:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776735.580152:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776735.580155:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776735.580158:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776735.580160:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776735.580161:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776735.580164:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776735.580166:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18450 00000400:00000200:0.0:1775776735.580170:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776735.580173:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776735.580175:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776735.580178:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580180:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776735.580189:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580191:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776735.580194:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776735.580195:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776735.580197:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776735.580199:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776735.580201:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776735.580202:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775776735.580203:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776735.580206:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775776735.580207:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776735.580210:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776735.580211:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776735.580214:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580218:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000040:2.0:1775776735.580219:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776735.580222:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776735.580229:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580233:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776735.580235:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776735.580236:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776735.580238:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776735.580240:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776735.580242:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.580242:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775776735.580245:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775776735.580246:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776735.580249:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776735.580253:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099788032:192.168.201.139@tcp:250:kworker.0 00000008:00100000:0.0:1775776735.580256:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1775776735.580257:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776735.580261:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1775776735.580263:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.580264:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580266:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.580268:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1775776735.580272:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 02000000:00000010:2.0:1775776735.580274:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 00000020:00000040:0.0:1775776735.580276:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776735.580277:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776735.580282:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776735.580284:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580285:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776735.580286:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776735.580288:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776735.580292:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000010:0.0:1775776735.580294:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000400:00000010:2.0:1775776735.580296:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000100:00000001:0.0:1775776735.580298:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776735.580304:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00100000:0.0:1775776735.580306:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775776735.580314:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099788032, portal 25 00000100:00000001:0.0:1775776735.580316:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776735.580318:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776735.580319:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.580321:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000100:00000040:0.0:1775776735.580325:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776735.580330:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776735.580346:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776735.580349:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776735.580352:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776735.580353:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776735.580356:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776735.580359:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776735.580360:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776735.580366:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18450 00000100:00000040:2.0:1775776735.580366:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776735.580371:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000400:00000200:0.0:1775776735.580372:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776735.580376:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776735.580377:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1775776735.580377:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099788032, offset 0 00000100:00000040:0.0:1775776735.580382:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580384:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776735.580389:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776735.580390:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580391:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776735.580394:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776735.580396:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776735.580397:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776735.580400:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000400:00000200:2.0:1775776735.580403:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:0.0:1775776735.580404:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776735.580407:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776735.580412:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776735.580412:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:0.0:1775776735.580417:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776735.580421:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776735.580426:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000040:0.0:1775776735.580427:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776735.580432:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776735.580435:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580436:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776735.580440:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4387:4387:256:4294967295] 192.168.201.139@tcp LPNI seq info [4387:4387:8:4294967295] 00000020:00000040:0.0:1775776735.580441:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776735.580448:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580451:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776735.580452:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776735.580455:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775776735.580456:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:0.0:1775776735.580459:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580461:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776735.580463:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776735.580467:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580470:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776735.580472:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776735.580478:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.580483:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000100:00000010:0.0:1775776735.580485:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776735.580489:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000200:2.0:1775776735.580491:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776735.580495:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:2.0:1775776735.580496:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 02000000:00000001:0.0:1775776735.580497:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580502:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776735.580505:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.1:1775776735.580505:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776735.580508:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776735.580509:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 02000000:00000010:0.0:1775776735.580514:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000800:00000100:2.0:1775776735.580515:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580519:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776735.580519:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000100:00100000:0.0:1775776735.580527:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776735.580527:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776735.580537:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775776735.580543:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776735.580546:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776735.580546:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776735.580550:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000040:0.0:1775776735.580554:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.580555:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775776735.580563:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776735.580566:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:2.0:1775776735.580568:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776735.580579:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776735.580579:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775776735.580583:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776735.580585:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1775776735.580585:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776735.580591:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776735.580597:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18450 00000100:00000001:2.0:1775776735.580598:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.580603:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1775776735.580606:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776735.580610:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776735.580612:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776735.580615:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580617:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776735.580617:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776735.580622:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580626:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776735.580626:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776735.580630:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313124597 00000020:00000001:0.0:1775776735.580631:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776735.580632:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776735.580634:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776735.580636:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776735.580638:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000010:2.0:1775776735.580638:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b8a0. 00000020:00000001:0.0:1775776735.580640:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776735.580644:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580645:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776735.580646:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580647:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776735.580647:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000020:00000040:0.0:1775776735.580651:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776735.580652:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580654:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776735.580655:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776735.580657:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776735.580657:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313124597 02000000:00000001:0.0:1775776735.580662:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580663:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000010:2.0:1775776735.580663:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b4e0. 00000008:00100000:0.0:1775776735.580667:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1775776735.580668:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1775776735.580669:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580671:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.580674:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1775776735.580676:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000100:00000001:2.0:1775776735.580676:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776735.580681:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00000001:2.0:1775776735.580681:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1775776735.580685:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1775776735.580686:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776735/real 1775776735] req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776735.580688:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580693:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776735.580695:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776735.580698:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775776735.580715:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776735.580716:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776735.580718:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776735.580722:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 02000000:00000010:0.0:1775776735.580723:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776735.580727:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776735.580734:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775776735.580734:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776735.580740:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1775776735.580744:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776735.580746:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776735.580747:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776735.580751:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776735.580771:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776735.580774:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:2.0:1775776735.580777:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776735.580811:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776735.580813:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776735.580815:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776735.580818:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776735.580824:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18450 00000400:00000200:0.0:1775776735.580830:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776735.580830:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776735.580834:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776735.580835:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776735.580840:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776735.580840:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.580844:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776735.580845:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580850:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1775776735.580854:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776735.580855:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580859:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776735.580864:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776735.580866:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776735.580867:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776735.580867:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775776735.580870:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776735.580874:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776735.580875:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776735.580877:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776735.580882:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580886:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776735.580889:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580892:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776735.580895:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00000001:2.0:1775776735.580895:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776735.580898:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776735.580901:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.580901:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775776735.580903:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776735.580905:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776735.580906:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776735.580908:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776735.580909:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776735.580912:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.580913:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776735.580914:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.580916:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1775776735.580918:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776735.580921:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:2.0:1775776735.580921:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1775776735.580923:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776735.580925:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.580925:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776735.580928:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776735.580930:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776735.580931:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776735.580931:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000010:0.0:1775776735.580936:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00080000:2.0:1775776735.580939:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776735.580940:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.580943:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1775776735.580946:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775776735.580951:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776735.580954:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776735.580957:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776735.580963:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776735.580970:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099788032:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776735.580973:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776735.580978:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18461) 00000100:00000001:2.0:1775776735.580978:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.580982:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.580987:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099788032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.580994:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776735.580997:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776735.581000:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776735.581003:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776735.581005:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776735.581007:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776735.581009:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776735.581011:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759dc00. 02000000:00000001:2.0:1775776735.581015:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776735.581017:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776735.581019:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:2.0:1775776735.581022:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776735.581022:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581024:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581030:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776735.581032:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581033:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776735.581035:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776735.581036:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776735.581039:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581045:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776735.581047:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.581048:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776735.581050:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776735.581052:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776735.581055:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788160:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776735.581058:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776735.581060:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776735.581061:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.581063:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776735.581066:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776735.581067:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776735.581071:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776735.581074:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776735.581077:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776735.581080:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099788160, portal 10 00000100:00000001:2.0:1775776735.581081:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776735.581083:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:2.0:1775776735.581088:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581094:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776735.581096:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776735.581097:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776735.581100:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099788160, offset 0 00000400:00000200:2.0:1775776735.581103:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776735.581106:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776735.581109:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776735.581112:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776735.581114:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776735.581116:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776735.581118:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776735.581121:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4388:4388:256:4294967295] 192.168.201.139@tcp LPNI seq info [4388:4388:8:4294967295] 00000400:00000200:2.0:1775776735.581124:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776735.581130:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776735.581134:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.581136:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776735.581140:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776735.581142:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776735.581146:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776735.581148:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776735.581151:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.581153:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776735.581155:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776735.581158:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776735.581161:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776735.581162:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776735.581164:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776735.581167:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581174:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.581177:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581189:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581190:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776735.581193:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776735.581196:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776735.581199:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776735.581200:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313123333 00000100:00000001:2.0:1775776735.581203:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313124597 00000100:00000001:2.0:1775776735.581205:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313123333 00000100:00000001:2.0:1775776735.581207:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313124597 00000100:00000001:2.0:1775776735.581209:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776735.581211:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581213:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776735.581216:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776735/real 1775776735] req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776735.581223:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776735.581224:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776735.581226:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776735.581228:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776735.581229:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776735.581231:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776735.581234:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776735.581241:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581247:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776735.581248:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581249:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.581250:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776735.581253:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581259:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776735.581263:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581268:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776735.581270:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581272:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.581273:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581274:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776735.581276:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776735.581278:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776735.581281:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776735.581283:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776735.581286:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776735.581293:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788160:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776735.581296:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776735.581299:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581305:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776735.581306:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.581307:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776735.581309:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776735.581311:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776735.581313:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788288:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776735.581315:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776735.581317:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776735.581319:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.581321:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776735.581323:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776735.581325:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776735.581328:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776735.581330:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776735.581332:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776735.581335:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099788288, portal 10 00000100:00000001:2.0:1775776735.581336:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776735.581338:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:2.0:1775776735.581342:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581348:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776735.581350:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776735.581351:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776735.581353:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099788288, offset 0 00000400:00000200:2.0:1775776735.581356:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776735.581359:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776735.581362:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776735.581364:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776735.581367:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776735.581369:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776735.581370:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776735.581373:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4389:4389:256:4294967295] 192.168.201.139@tcp LPNI seq info [4389:4389:8:4294967295] 00000400:00000200:2.0:1775776735.581376:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776735.581381:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776735.581386:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.581388:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776735.581392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776735.581394:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776735.581398:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776735.581400:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776735.581403:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.581405:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776735.581408:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776735.581411:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776735.581413:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776735.581414:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776735.581416:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776735.581418:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581424:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.581426:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581432:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581433:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776735.581435:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776735.581437:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776735.581439:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776735.581441:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313123334 00000100:00000001:2.0:1775776735.581443:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313124597 00000100:00000001:2.0:1775776735.581444:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313123334 00000100:00000001:2.0:1775776735.581446:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313124597 00000100:00000001:2.0:1775776735.581447:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776735.581449:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581451:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776735.581453:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776735/real 1775776735] req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776735.581459:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776735.581460:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776735.581461:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776735.581463:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776735.581464:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776735.581466:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776735.581468:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776735.581475:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581480:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776735.581482:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581483:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.581484:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776735.581487:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581493:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776735.581497:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581502:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776735.581504:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581505:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.581506:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581508:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776735.581510:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776735.581512:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776735.581514:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776735.581516:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776735.581519:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776735.581525:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788288:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776735.581528:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776735.581531:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581536:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776735.581538:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.581539:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776735.581541:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776735.581542:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776735.581545:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788416:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776735.581548:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776735.581549:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776735.581551:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.581552:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776735.581555:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776735.581556:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776735.581559:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776735.581561:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776735.581564:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776735.581566:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099788416, portal 4 00000100:00000001:2.0:1775776735.581568:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776735.581569:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:2.0:1775776735.581574:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581580:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776735.581582:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776735.581584:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776735.581586:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099788416, offset 0 00000400:00000200:2.0:1775776735.581590:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776735.581593:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776735.581596:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776735.581599:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776735.581601:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776735.581603:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776735.581605:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776735.581608:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4390:4390:256:4294967295] 192.168.201.139@tcp LPNI seq info [4390:4390:8:4294967295] 00000400:00000200:2.0:1775776735.581612:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776735.581618:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776735.581622:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.581625:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776735.581628:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776735.581631:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776735.581635:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776735.581637:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776735.581640:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.581642:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776735.581644:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776735.581647:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776735.581650:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776735.581652:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776735.581654:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776735.581656:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581662:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.581665:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581670:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581671:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776735.581674:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776735.581676:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776735.581679:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776735.581681:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313123343 00000100:00000001:2.0:1775776735.581683:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313124597 00000100:00000001:2.0:1775776735.581685:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313123343 00000100:00000001:2.0:1775776735.581687:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313124597 00000100:00000001:2.0:1775776735.581689:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776735.581691:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581693:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776735.581696:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776735/real 1775776735] req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776735.581714:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776735.581715:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776735.581716:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776735.581718:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776735.581719:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776735.581720:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776735.581722:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776735.581727:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581731:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776735.581732:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581732:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.581733:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776735.581735:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581739:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776735.581741:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581745:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776735.581746:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581747:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.581748:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581749:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776735.581750:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776735.581751:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776735.581753:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776735.581755:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776735.581757:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776735.581764:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788416:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776735.581767:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776735.581770:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581775:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776735.581776:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.581799:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776735.581801:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776735.581803:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776735.581806:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788544:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776735.581809:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776735.581811:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776735.581812:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776735.581814:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776735.581817:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776735.581819:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776735.581823:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776735.581825:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776735.581828:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776735.581830:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099788544, portal 4 00000100:00000001:2.0:1775776735.581832:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776735.581833:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:2.0:1775776735.581839:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581844:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776735.581846:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776735.581847:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776735.581849:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099788544, offset 0 00000400:00000200:2.0:1775776735.581851:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776735.581854:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776735.581856:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776735.581858:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776735.581859:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776735.581861:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776735.581862:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776735.581864:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4391:4391:256:4294967295] 192.168.201.139@tcp LPNI seq info [4391:4391:8:4294967295] 00000400:00000200:2.0:1775776735.581867:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776735.581870:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776735.581873:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.581874:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776735.581877:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776735.581879:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776735.581881:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776735.581882:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776735.581885:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776735.581886:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776735.581888:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776735.581889:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776735.581891:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776735.581892:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776735.581894:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776735.581895:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581900:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.581901:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776760 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581905:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581906:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776735.581908:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776735.581910:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776735.581912:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776735.581913:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313123335 00000100:00000001:2.0:1775776735.581914:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313124597 00000100:00000001:2.0:1775776735.581915:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313123335 00000100:00000001:2.0:1775776735.581916:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313124597 00000100:00000001:2.0:1775776735.581917:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776735.581919:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.581920:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776735.581921:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776735/real 1775776735] req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776735.581926:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776735.581926:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776735.581927:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776735.581929:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776735.581930:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776735.581931:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776735.581932:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776735.581936:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581940:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776735.581941:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581942:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.581943:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776735.581945:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581948:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776735.581950:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581954:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776735.581955:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581956:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776735.581957:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581958:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776735.581959:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776735.581960:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776735.581962:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776735.581963:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776735.581965:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776735.581969:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788544:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776735.581971:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581972:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.581973:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099788160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581977:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776735.581978:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776735.581979:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776735.581980:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776735.581981:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776735.581982:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776735.581983:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776735.581984:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d000. 02000000:00000001:2.0:1775776735.581986:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776735.581987:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776735.581987:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000100:00000001:2.0:1775776735.581989:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776735.581990:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.581991:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.581993:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099788288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.581996:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776735.581997:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776735.581998:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776735.581999:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776735.581999:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776735.582000:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776735.582001:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776735.582001:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:2.0:1775776735.582002:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776735.582003:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776735.582004:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:2.0:1775776735.582005:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776735.582005:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.582006:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.582008:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099788416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.582011:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776735.582011:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776735.582012:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776735.582013:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776735.582013:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776735.582014:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776735.582015:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776735.582016:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:2.0:1775776735.582017:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776735.582017:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776735.582018:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:2.0:1775776735.582019:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776735.582020:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.582020:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776735.582022:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099788544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776760 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776735.582029:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776735.582029:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776735.582030:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776735.582031:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776735.582032:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776735.582032:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776735.582033:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776735.582034:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:2.0:1775776735.582036:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776735.582036:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776735.582037:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:2.0:1775776735.582038:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776735.582039:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.582039:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776735.582042:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776735.582043:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776735.582044:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776735.582045:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776736.283986:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776736.283993:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776736.283995:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776736.284000:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776736.284003:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776736.284007:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776736.284010:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776736.284012:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776736.284015:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776736.347907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776736.347914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776736.347915:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776736.347920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776736.347921:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776736.347923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776736.347924:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776736.347929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776736.347930:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776736.347931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776736.347936:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776736.347938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776736.347940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776736.347940:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776736.347945:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776736.347947:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776736.347949:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776736.348006:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776736.348010:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776736.348013:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776736.859892:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313124597, at 4313124608 00000100:00000010:1.0:1775776736.859908:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000040:0.0:1775776736.859913:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313124597, at 4313124608 00000100:00000010:0.0:1775776736.859922:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000001:3.0:1775776737.308049:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776737.308056:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776737.308058:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776737.308064:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776737.308068:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776737.308072:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776737.308075:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776737.308077:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776737.308080:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776737.371899:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776737.371904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776737.371907:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776737.371908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776737.371910:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776737.371915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776737.371928:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776737.371933:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776737.371939:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776737.371941:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776737.371944:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776737.371946:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776737.372022:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776737.372028:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776737.372029:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776737.372032:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776737.372033:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776737.372436:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776737.372440:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776737.372443:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776738.335677:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776738.335684:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776738.335686:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776738.335692:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776738.335696:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776738.335709:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776738.335712:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776738.335717:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776738.335720:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776738.395909:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776738.395918:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776738.395921:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776738.395924:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776738.395928:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776738.395930:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776738.395932:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776738.395935:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776738.395939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776738.395940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776738.395943:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776738.395944:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776738.395948:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776738.395949:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776738.396007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776738.396010:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776738.396012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776738.399807:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776738.399815:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776738.399817:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776738.399822:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776738.399826:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776738.399830:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776738.399833:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776738.399836:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776738.399839:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776739.355905:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776739.355913:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776739.355916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776739.355921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776739.355925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776739.355929:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776739.355932:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776739.355934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776739.355937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776739.419883:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776739.419888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776739.419891:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776739.419893:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776739.419895:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776739.419899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776739.419901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776739.419901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776739.419903:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776739.419905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776739.419906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776739.419908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776739.419910:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776739.419910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776739.419914:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776739.419916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776739.419919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776739.419969:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776739.419973:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776739.419976:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776740.380162:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776740.380169:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776740.380171:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776740.380177:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776740.380179:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776740.380183:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776740.380184:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776740.380186:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776740.380188:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.443902:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776740.443909:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776740.443911:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776740.443917:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776740.443920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776740.443925:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776740.443927:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776740.443929:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776740.443932:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776740.444610:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776740.444614:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776740.444629:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776740.444630:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776740.444633:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776740.444635:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776740.444638:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776740.444639:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776740.444716:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776740.444719:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776740.444722:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776740.699923:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776740.699933:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776740.699936:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776740.699939:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776740.699942:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776740.699948:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18456 00000400:00000200:3.0:1775776740.699957:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776740.699962:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776740.699964:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776740.699966:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.699969:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776740.699971:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.699972:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776740.699975:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776740.699976:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776740.699978:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776740.699980:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776740.699981:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776740.699983:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776740.699984:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.699986:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776740.699987:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.699988:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776740.699989:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776740.699992:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776740.699994:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776740.699996:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776740.699998:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776740.700000:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700001:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776740.700003:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776740.700010:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776740.700013:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776740.700015:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776740.700017:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700018:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776740.700021:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776740.700024:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776740.700029:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776740.700035:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776740.700041:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776740.700048:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776740.700052:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776740.700055:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776740.700071:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776740.700073:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776740.700075:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776740.700077:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776740.700078:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776740.700079:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776740.700081:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18456 00000400:00000200:3.0:1775776740.700082:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776740.700084:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776740.700085:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776740.700087:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700089:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776740.700090:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700091:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776740.700092:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776740.700093:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776740.700094:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776740.700096:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776740.700097:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776740.700098:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776740.700100:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700102:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776740.700103:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700104:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776740.700105:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776740.700107:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700109:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776740.700110:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776740.700112:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1775776740.700113:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776740.700117:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700118:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776740.700118:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776740.700122:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1775776740.700123:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776740.700126:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700127:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776740.700130:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000100:00000040:2.0:1775776740.700131:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776740.700133:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776740.700137:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776740.700139:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700142:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776740.700144:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776740.700146:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700151:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000010:3.0:1775776740.700152:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:2.0:1775776740.700154:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700155:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.700159:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1775776740.700162:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775776740.700163:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776740.700169:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700172:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1775776740.700175:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099788672:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776740.700178:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776740.700183:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776740.700189:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700192:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1775776740.700192:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1775776740.700197:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776740.700200:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776740.700202:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:2.0:1775776740.700204:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776740.700205:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776740.700208:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776740.700210:0:313512:0:(import.c:484:import_select_connection()) Process entered 02000000:00000010:2.0:1775776740.700211:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 00000100:00080000:3.0:1775776740.700214:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18456 02000000:00000001:2.0:1775776740.700215:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776740.700221:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.700225:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776740.700227:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000010:2.0:1775776740.700231:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000100:00080000:3.0:1775776740.700231:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776740.700237:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700239:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1775776740.700241:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000040:3.0:1775776740.700242:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700246:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776740.700249:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776740.700250:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776740.700252:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776740.700254:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776740.700256:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776740.700257:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776740.700260:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000200:2.0:1775776740.700262:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099788672, portal 25 00000100:00000001:3.0:1775776740.700262:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776740.700265:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776740.700267:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776740.700268:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776740.700271:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751254400 : -114841958297216 : ffff978d45ded180) 00000020:00000040:3.0:1775776740.700271:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776740.700275:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700278:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776740.700280:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1775776740.700282:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776740.700283:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776740.700287:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700290:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776740.700292:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776740.700297:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700299:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776740.700299:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776740.700304:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000100:00000040:2.0:1775776740.700305:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000020:00000040:3.0:1775776740.700308:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000400:00000010:2.0:1775776740.700310:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 02000000:00000001:3.0:1775776740.700311:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776740.700313:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700314:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776740.700315:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776740.700318:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776740.700321:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099788672, offset 0 02000000:00000010:3.0:1775776740.700324:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776740.700328:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776740.700333:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776740.700343:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700343:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776740.700347:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776740.700352:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776740.700356:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776740.700364:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776740.700366:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00080000:3.0:1775776740.700368:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775776740.700372:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775776740.700373:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776740.700376:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776740.700378:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775776740.700380:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000001:3.0:1775776740.700382:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776740.700387:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18456 00000400:00000200:2.0:1775776740.700395:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776740.700399:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776740.700402:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4392:4392:256:4294967295] 192.168.201.139@tcp LPNI seq info [4392:4392:8:4294967295] 00000400:00000200:2.0:1775776740.700407:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776740.700414:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776740.700419:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776740.700422:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775776740.700425:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000040:3.0:1775776740.700428:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700431:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776740.700433:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700436:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000800:00000200:2.0:1775776740.700438:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000020:00000001:3.0:1775776740.700439:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776740.700440:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776740.700443:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776740.700446:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000800:00000010:2.0:1775776740.700448:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000020:00000001:3.0:1775776740.700448:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776740.700453:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000800:00000200:2.0:1775776740.700456:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:3.0:1775776740.700458:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:2.0:1775776740.700460:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1775776740.700463:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1775776740.700468:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:3.0:1775776740.700469:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776740.700473:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1775776740.700474:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776740.700477:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776740.700479:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700483:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776740.700486:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:2.0:1775776740.700490:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 02000000:00000001:3.0:1775776740.700490:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776740.700495:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776740.700496:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000100:00000001:3.0:1775776740.700498:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776740.700501:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776740.700503:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776740.700504:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776740.700504:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776740.700511:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000400:00000200:2.0:1775776740.700512:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000020:00000040:3.0:1775776740.700516:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776740.700519:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1775776740.700520:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 02000000:00000001:3.0:1775776740.700521:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776740.700524:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:3.0:1775776740.700524:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776740.700527:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776740.700530:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700531:0:301888:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:3.0:1775776740.700534:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000200:2.0:1775776740.700538:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776740.700539:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700550:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1775776740.700550:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775776740.700556:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776740.700568:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776740.700569:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700570:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700572:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775776740.700576:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.700578:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776740.700585:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776740.700585:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700589:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313129717 00000100:00080000:3.0:1775776740.700589:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776740.700594:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776740.700596:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000010:2.0:1775776740.700597:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b3c0. 00000100:00080000:3.0:1775776740.700597:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776740.700601:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776740.700604:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18456 00000100:00000001:2.0:1775776740.700606:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000400:00000200:3.0:1775776740.700610:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776740.700624:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776740.700626:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776740.700628:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700630:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776740.700632:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700634:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776740.700636:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776740.700637:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776740.700639:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776740.700641:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776740.700642:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776740.700644:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:2.0:1775776740.700646:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313129717 00000100:00000040:3.0:1775776740.700647:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700652:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776740.700653:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000010:2.0:1775776740.700654:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b8a0. 00000100:00000001:3.0:1775776740.700657:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776740.700659:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000020:00000040:3.0:1775776740.700660:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776740.700665:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776740.700667:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776740.700669:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776740.700672:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776740.700673:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776740.700674:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700677:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700678:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776740.700680:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000008:00100000:3.0:1775776740.700681:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1775776740.700686:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776740/real 1775776740] req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775776740.700687:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776740.700691:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776740.700699:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000400:00000001:2.0:1775776740.700700:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776740.700703:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:3.0:1775776740.700703:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000400:00000010:2.0:1775776740.700705:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776740.700709:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 02000000:00000001:3.0:1775776740.700709:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776740.700711:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776740.700712:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:3.0:1775776740.700714:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775776740.700715:0:301888:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1775776740.700716:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776740.700719:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776740.700722:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775776740.700723:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776740.700728:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776740.700737:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775776740.700741:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776740.700752:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776740.700756:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776740.700756:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700759:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776740.700762:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.700763:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.700767:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776740.700776:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776740.700807:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776740.700812:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18466) 00000100:00000001:2.0:1775776740.700821:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776740.700827:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.700835:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776740.700838:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.700840:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.700841:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.700844:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776740.700846:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776740.700848:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776740.700851:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776740.700852:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776740.700855:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776740.700863:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099788672:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776740.700868:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.700870:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.700872:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099788672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.700878:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776740.700880:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776740.700883:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776740.700885:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776740.700887:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776740.700889:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776740.700890:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776740.700893:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:2.0:1775776740.700896:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776740.700898:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776740.700899:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:2.0:1775776740.700902:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776740.700903:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.700906:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.700912:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776740.700914:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.700916:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776740.700916:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776740.700917:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776740.700920:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.700925:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776740.700926:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.700927:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776740.700928:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776740.700930:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776740.700931:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776740.700933:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776740.700935:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776740.700936:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.700937:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776740.700939:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776740.700940:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776740.700943:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776740.700945:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776740.700947:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776740.700950:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099788800, portal 10 00000100:00000001:2.0:1775776740.700952:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776740.700953:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751260672 : -114841958290944 : ffff978d45deea00) 00000100:00000040:2.0:1775776740.700959:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.700965:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776740.700967:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776740.700969:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776740.700971:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099788800, offset 0 00000400:00000200:2.0:1775776740.700976:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776740.700988:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.700992:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776740.700995:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776740.700997:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776740.700999:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776740.701003:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776740.701005:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4393:4393:256:4294967295] 192.168.201.139@tcp LPNI seq info [4393:4393:8:4294967295] 00000400:00000200:2.0:1775776740.701007:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.701011:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776740.701015:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776740.701017:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776740.701020:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776740.701021:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776740.701025:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776740.701026:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776740.701028:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776740.701030:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776740.701032:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776740.701034:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776740.701036:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776740.701037:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776740.701038:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776740.701040:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701046:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.701047:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701051:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701052:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776740.701054:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776740.701056:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776740.701058:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776740.701059:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313128453 00000100:00000001:2.0:1775776740.701061:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313129717 00000100:00000001:2.0:1775776740.701062:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313128453 00000100:00000001:2.0:1775776740.701064:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313129717 00000100:00000001:2.0:1775776740.701065:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776740.701066:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701067:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776740.701069:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776740/real 1775776740] req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776740.701074:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776740.701075:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776740.701076:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776740.701077:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776740.701078:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776740.701079:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776740.701081:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776740.701086:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701089:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776740.701090:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701091:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.701091:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776740.701093:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701097:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776740.701099:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701102:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776740.701103:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701104:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.701105:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701106:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776740.701107:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776740.701108:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776740.701110:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776740.701111:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776740.701113:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776740.701117:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776740.701119:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776740.701121:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701129:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776740.701130:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.701131:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776740.701132:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776740.701133:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776740.701135:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776740.701136:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776740.701137:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776740.701138:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.701139:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776740.701141:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776740.701142:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776740.701144:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776740.701145:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776740.701147:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776740.701148:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099788928, portal 10 00000100:00000001:2.0:1775776740.701149:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776740.701150:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751250816 : -114841958300800 : ffff978d45dec380) 00000100:00000040:2.0:1775776740.701153:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701157:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776740.701158:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776740.701160:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776740.701161:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099788928, offset 0 00000400:00000200:2.0:1775776740.701163:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776740.701165:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.701167:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776740.701168:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776740.701170:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776740.701171:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776740.701172:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776740.701174:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4394:4394:256:4294967295] 192.168.201.139@tcp LPNI seq info [4394:4394:8:4294967295] 00000400:00000200:2.0:1775776740.701177:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.701180:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776740.701183:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776740.701184:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776740.701186:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776740.701188:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776740.701192:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776740.701195:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776740.701198:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776740.701200:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776740.701202:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776740.701205:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776740.701208:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776740.701209:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776740.701211:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776740.701214:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701221:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.701224:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701230:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701232:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776740.701235:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776740.701237:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776740.701240:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776740.701242:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313128454 00000100:00000001:2.0:1775776740.701244:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313129717 00000100:00000001:2.0:1775776740.701246:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313128454 00000100:00000001:2.0:1775776740.701248:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313129717 00000100:00000001:2.0:1775776740.701249:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776740.701251:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701252:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776740.701254:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776740/real 1775776740] req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776740.701259:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776740.701260:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776740.701261:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776740.701263:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776740.701263:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776740.701265:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776740.701266:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776740.701271:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701274:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776740.701275:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701276:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.701277:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776740.701279:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701282:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776740.701284:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701287:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776740.701288:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701289:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.701290:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701291:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776740.701292:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776740.701294:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776740.701296:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776740.701297:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776740.701299:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776740.701303:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099788928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776740.701305:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776740.701307:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701310:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776740.701311:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.701313:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776740.701314:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776740.701316:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776740.701318:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776740.701321:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776740.701323:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776740.701324:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.701325:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776740.701328:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776740.701330:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776740.701333:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776740.701335:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776740.701338:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776740.701341:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099789056, portal 4 00000100:00000001:2.0:1775776740.701342:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776740.701344:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751252608 : -114841958299008 : ffff978d45deca80) 00000100:00000040:2.0:1775776740.701349:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701355:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776740.701358:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776740.701360:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776740.701363:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099789056, offset 0 00000400:00000200:2.0:1775776740.701367:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776740.701371:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.701374:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776740.701376:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776740.701377:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776740.701379:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776740.701380:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776740.701383:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4395:4395:256:4294967295] 192.168.201.139@tcp LPNI seq info [4395:4395:8:4294967295] 00000400:00000200:2.0:1775776740.701386:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.701392:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776740.701396:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776740.701398:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776740.701402:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776740.701404:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776740.701408:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776740.701410:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776740.701414:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776740.701415:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776740.701418:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776740.701421:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776740.701424:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776740.701426:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776740.701428:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776740.701431:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701438:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.701441:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701445:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701446:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776740.701448:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776740.701450:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776740.701452:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776740.701454:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313128463 00000100:00000001:2.0:1775776740.701455:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313129717 00000100:00000001:2.0:1775776740.701456:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313128463 00000100:00000001:2.0:1775776740.701458:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313129717 00000100:00000001:2.0:1775776740.701459:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776740.701460:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701461:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776740.701463:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776740/real 1775776740] req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776740.701468:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776740.701468:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776740.701469:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776740.701471:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776740.701472:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776740.701473:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776740.701475:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776740.701479:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701482:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776740.701483:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701484:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.701485:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776740.701488:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701493:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776740.701496:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701502:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776740.701504:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701505:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.701507:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701508:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776740.701510:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776740.701512:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776740.701515:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776740.701517:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776740.701521:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776740.701529:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776740.701532:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776740.701535:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701540:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776740.701541:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.701543:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776740.701544:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776740.701545:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776740.701547:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776740.701549:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776740.701550:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776740.701551:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776740.701552:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776740.701554:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776740.701555:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776740.701557:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776740.701559:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776740.701561:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776740.701563:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099789184, portal 4 00000100:00000001:2.0:1775776740.701564:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776740.701565:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751259776 : -114841958291840 : ffff978d45dee680) 00000100:00000040:2.0:1775776740.701568:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701572:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776740.701574:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776740.701575:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776740.701576:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099789184, offset 0 00000400:00000200:2.0:1775776740.701578:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776740.701581:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.701582:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776740.701584:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776740.701586:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776740.701587:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776740.701588:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776740.701591:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4396:4396:256:4294967295] 192.168.201.139@tcp LPNI seq info [4396:4396:8:4294967295] 00000400:00000200:2.0:1775776740.701593:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776740.701596:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776740.701598:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776740.701600:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776740.701603:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776740.701604:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776740.701607:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776740.701608:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776740.701611:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776740.701626:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776740.701629:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776740.701632:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776740.701634:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776740.701636:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776740.701639:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776740.701642:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701649:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.701651:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776765 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701657:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701658:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776740.701661:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776740.701664:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776740.701667:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776740.701669:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313128454 00000100:00000001:2.0:1775776740.701671:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313129717 00000100:00000001:2.0:1775776740.701673:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313128454 00000100:00000001:2.0:1775776740.701675:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313129717 00000100:00000001:2.0:1775776740.701677:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776740.701679:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701681:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776740.701684:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776740/real 1775776740] req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776740.701689:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776740.701689:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776740.701690:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776740.701692:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776740.701693:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776740.701694:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776740.701696:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776740.701700:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701707:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776740.701709:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701710:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.701711:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776740.701712:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701716:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776740.701718:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701723:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776740.701725:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701727:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776740.701727:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701729:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776740.701730:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776740.701733:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776740.701735:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776740.701737:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776740.701741:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776740.701749:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776740.701752:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701753:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.701756:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099788800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701763:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776740.701764:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776740.701766:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776740.701769:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776740.701770:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776740.701771:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776740.701773:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776740.701775:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:2.0:1775776740.701777:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776740.701796:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776740.701798:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:2.0:1775776740.701802:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776740.701803:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701805:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.701808:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec380 x1862033099788928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701814:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776740.701815:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776740.701817:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776740.701819:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776740.701820:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776740.701821:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776740.701823:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776740.701825:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:2.0:1775776740.701827:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776740.701828:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776740.701829:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec380. 00000100:00000001:2.0:1775776740.701831:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776740.701832:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701834:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.701837:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deca80 x1862033099789056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701844:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776740.701845:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776740.701846:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776740.701847:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776740.701847:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776740.701848:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776740.701850:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776740.701851:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:2.0:1775776740.701852:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776740.701853:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776740.701854:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deca80. 00000100:00000001:2.0:1775776740.701855:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776740.701856:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701856:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776740.701859:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee680 x1862033099789184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776765 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776740.701864:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776740.701865:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776740.701866:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776740.701868:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776740.701869:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776740.701870:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776740.701871:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776740.701873:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:2.0:1775776740.701874:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776740.701875:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776740.701876:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee680. 00000100:00000001:2.0:1775776740.701878:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776740.701879:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701881:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776740.701884:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776740.701886:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776740.701887:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776740.701889:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776741.405675:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776741.405683:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776741.405685:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776741.405689:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776741.405693:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776741.405696:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776741.405698:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776741.405701:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776741.405704:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776741.467908:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776741.467909:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776741.467914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776741.467921:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776741.467923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776741.467924:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467927:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00000100:00100000:3.0:1775776741.467929:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776741.467931:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1775776741.467935:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776741.467936:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776741.467939:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776741.467940:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776741.467942:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467945:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467947:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776741.467950:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776741.467953:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1775776741.467954:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776741.467955:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776741.467956:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467959:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776741.467960:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776741.467960:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467964:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467967:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776741.467970:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776741.467971:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776741.467972:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467973:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776741.467973:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467974:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776741.467975:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776741.468036:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776741.468040:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776741.468043:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776741.979885:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313129717, at 4313129728 00000100:00000040:1.0:1775776741.979892:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313129717, at 4313129728 00000100:00000010:0.0:1775776741.979900:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000010:1.0:1775776741.979901:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000001:3.0:1775776742.427934:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776742.427941:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776742.427943:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776742.427949:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776742.427953:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776742.427957:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776742.427960:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776742.427962:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776742.427964:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776742.491903:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776742.491908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776742.491914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776742.491916:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776742.491920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776742.491921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776742.491925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776742.491927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776742.491969:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776742.491974:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776742.491976:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776742.491981:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776742.491985:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776742.491988:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776742.491991:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1775776742.491993:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000001:3.0:1775776742.491994:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1775776742.491999:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776742.491999:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776742.492004:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776743.451911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776743.451916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776743.451918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776743.451922:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776743.451924:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776743.451927:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776743.451929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776743.451931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776743.451932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776743.515842:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776743.515849:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776743.515852:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776743.515857:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776743.515861:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776743.515864:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776743.515867:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776743.515870:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776743.515873:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776743.515976:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776743.515980:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776743.515984:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776743.515986:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776743.515989:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776743.515990:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776743.515993:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776743.515994:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776743.516111:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776743.516115:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776743.516118:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776744.475891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776744.475898:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776744.475900:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776744.475913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776744.475917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776744.475921:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776744.475923:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776744.475926:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776744.475928:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776744.539878:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776744.539883:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776744.539884:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776744.539889:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776744.539892:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776744.539895:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776744.539897:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776744.539899:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776744.539900:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776744.540175:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776744.540178:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776744.540181:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776744.540182:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776744.540184:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776744.540185:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776744.540187:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776744.540188:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776744.540253:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776744.540257:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776744.540259:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776745.499902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776745.499911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776745.499913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776745.499918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776745.499922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776745.499926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776745.499929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776745.499932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776745.499934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776745.563871:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776745.563876:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776745.563883:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776745.563884:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776745.563888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776745.563889:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776745.563892:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776745.563893:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776745.564157:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776745.564162:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776745.564164:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776745.564168:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776745.564171:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776745.564174:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776745.564176:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776745.564178:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776745.564181:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776745.564224:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776745.564226:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776745.564228:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776745.820409:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776745.820419:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776745.820422:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776745.820426:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776745.820430:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776745.820436:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18461 00000400:00000200:0.0:1775776745.820445:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776745.820451:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776745.820454:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776745.820457:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820460:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776745.820462:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820464:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776745.820467:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776745.820468:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776745.820470:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776745.820472:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776745.820473:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776745.820475:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776745.820477:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820478:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776745.820479:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820480:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776745.820482:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776745.820484:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820485:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776745.820487:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776745.820489:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776745.820491:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.820492:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776745.820494:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776745.820502:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776745.820504:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776745.820507:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776745.820508:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.820510:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776745.820511:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776745.820514:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776745.820521:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776745.820542:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776745.820555:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776745.820565:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776745.820569:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776745.820573:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776745.820607:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776745.820610:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776745.820614:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776745.820615:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776745.820617:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776745.820620:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776745.820622:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18461 00000400:00000200:0.0:1775776745.820626:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776745.820629:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776745.820631:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776745.820634:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820637:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776745.820638:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820640:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776745.820642:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776745.820643:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776745.820645:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776745.820646:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776745.820648:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776745.820650:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776745.820652:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820654:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776745.820655:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820657:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776745.820658:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776745.820661:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776745.820663:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776745.820664:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776745.820668:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776745.820670:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.820672:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776745.820743:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776745.820748:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776745.820751:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776745.820757:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.820776:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776745.820801:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.820805:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776745.820807:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776745.820810:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776745.820814:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099789312:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776745.820818:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776745.820822:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776745.820824:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.820827:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776745.820831:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776745.820833:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776745.820839:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776745.820843:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776745.820849:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776745.820856:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099789312, portal 25 00000100:00000001:2.0:1775776745.820858:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776745.820860:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757056 : -114841967794560 : ffff978d454de680) 00000100:00000040:2.0:1775776745.820865:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.820874:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776745.820877:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776745.820879:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776745.820883:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099789312, offset 0 00000400:00000200:2.0:1775776745.820889:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776745.820896:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.820902:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776745.820907:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776745.820911:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776745.820915:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776745.820918:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776745.820923:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4397:4397:256:4294967295] 192.168.201.139@tcp LPNI seq info [4397:4397:8:4294967295] 00000400:00000200:2.0:1775776745.820928:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000008:00100000:0.0:1775776745.820929:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776745.820933:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.820937:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776745.820941:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000010:0.0:1775776745.820944:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775776745.820947:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 00000800:00000200:2.0:1775776745.820949:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 02000000:00000001:0.0:1775776745.820951:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776745.820953:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776745.820955:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000100:00000001:0.0:1775776745.820956:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776745.820958:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776745.820961:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776745.820965:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000010:0.0:1775776745.820969:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000800:00000200:2.0:1775776745.820970:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1775776745.820974:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776745.820978:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776745.820982:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00100000:0.0:1775776745.820983:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775776745.820990:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776745.820993:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000100:00000001:0.0:1775776745.820999:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775776745.821001:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775776745.821002:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776745.821009:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776745.821009:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776745.821014:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776745.821018:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776745.821023:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775776745.821024:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776745.821028:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:2.0:1775776745.821030:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776745.821035:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776745.821039:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776745.821042:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776745.821046:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776745.821046:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1775776745.821050:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18461 00000100:00000040:2.0:1775776745.821052:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776745.821059:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776745.821066:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776745.821068:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776745.821072:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821074:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776745.821078:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821083:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776745.821086:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776745.821087:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776745.821090:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000100:00000001:2.0:1775776745.821090:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1775776745.821093:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776745.821096:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000100:00000001:2.0:1775776745.821097:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776745.821099:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776745.821103:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776745.821103:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776745.821107:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313134838 00000100:00000040:0.0:1775776745.821110:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821113:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1775776745.821114:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b4e0. 00000100:00000040:0.0:1775776745.821116:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821120:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776745.821121:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000020:00000040:0.0:1775776745.821123:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776745.821127:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821130:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821131:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313134838 02000000:00000001:0.0:1775776745.821132:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776745.821137:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1775776745.821137:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b7e0. 02000000:00000001:0.0:1775776745.821142:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821143:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776745.821144:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776745.821148:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1775776745.821148:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775776745.821154:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821155:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.821157:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821159:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000010:0.0:1775776745.821164:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000100:00100000:2.0:1775776745.821165:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776745/real 1775776745] req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776745.821169:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776745.821175:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776745.821177:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.821180:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:2.0:1775776745.821181:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1775776745.821182:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:2.0:1775776745.821184:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1775776745.821185:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776745.821187:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 02000000:00000010:0.0:1775776745.821191:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000400:00000200:2.0:1775776745.821191:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776745.821195:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:0.0:1775776745.821196:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821198:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1775776745.821205:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775776745.821206:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776745.821221:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776745.821224:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776745.821232:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776745.821235:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776745.821262:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821262:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776745.821265:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776745.821266:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776745.821268:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.821271:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776745.821272:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775776745.821276:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776745.821285:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776745.821288:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776745.821288:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776745.821293:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:2.0:1775776745.821295:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776745.821299:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18461 00000400:00000200:0.0:1775776745.821306:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776745.821309:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776745.821314:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776745.821316:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1775776745.821319:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776745.821320:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776745.821324:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776745.821327:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821329:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776745.821331:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775776745.821332:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1775776745.821333:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000100:00000001:2.0:1775776745.821335:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1775776745.821336:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776745.821338:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000020:00000040:0.0:1775776745.821339:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000100:00000001:2.0:1775776745.821341:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1775776745.821343:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00080000:2.0:1775776745.821346:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:0.0:1775776745.821347:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776745.821352:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00080000:2.0:1775776745.821354:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776745.821355:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776745.821360:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776745.821361:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821365:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000040:2.0:1775776745.821368:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776745.821369:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776745.821376:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821382:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776745.821386:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775776745.821386:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099789312:192.168.201.139@tcp:250:kworker.0 02000000:00000001:0.0:1775776745.821390:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776745.821392:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775776745.821394:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821395:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775776745.821399:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:2.0:1775776745.821400:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de680 x1862033099789312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775776745.821402:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776745.821408:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.821412:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821414:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776745.821417:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000010:0.0:1775776745.821418:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 02000000:00000010:2.0:1775776745.821420:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 00000020:00000040:0.0:1775776745.821422:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776745.821424:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1775776745.821426:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000020:00000001:2.0:1775776745.821427:0:301888:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:0.0:1775776745.821429:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776745.821430:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821434:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1775776745.821434:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:0.0:1775776745.821438:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:2.0:1775776745.821438:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776745.821441:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776745.821446:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 02000000:00000001:2.0:1775776745.821449:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1775776745.821450:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.821452:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776745.821455:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de680. 00000100:00100000:0.0:1775776745.821458:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776745.821460:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776745.821462:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.821465:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776745.821472:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776745.821474:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776745.821475:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821476:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821480:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776745.821482:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821483:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776745.821485:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776745.821491:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776745.821500:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821504:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775776745.821505:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776745.821507:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.821511:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1775776745.821512:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:2.0:1775776745.821513:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776745.821516:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776745.821519:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789440:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776745.821522:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776745.821532:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1775776745.821533:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1775776745.821535:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1775776745.821535:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.821539:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776745.821540:0:303560:0:(import.c:484:import_select_connection()) Process entered 02000000:00000010:2.0:1775776745.821542:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 00000100:00080000:0.0:1775776745.821545:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18461 02000000:00000001:2.0:1775776745.821546:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776745.821553:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.821554:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776745.821559:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000200:0.0:1775776745.821560:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776745.821564:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000010:2.0:1775776745.821565:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000040:0.0:1775776745.821570:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000200:2.0:1775776745.821570:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099789440, portal 10 00000100:00000001:2.0:1775776745.821573:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776745.821574:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776745.821576:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741762432 : -114841967789184 : ffff978d454dfb80) 00000100:00000040:0.0:1775776745.821579:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821582:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000040:2.0:1775776745.821585:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821592:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776745.821594:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776745.821597:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776745.821599:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099789440, offset 0 00000400:00000200:2.0:1775776745.821602:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776745.821606:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.821618:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776745.821621:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776745.821623:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776745.821625:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776745.821628:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776745.821638:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4398:4398:256:4294967295] 192.168.201.139@tcp LPNI seq info [4398:4398:8:4294967295] 00000400:00000200:2.0:1775776745.821642:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.821648:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776745.821652:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000020:00000001:0.0:1775776745.821655:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776745.821657:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776745.821659:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776745.821661:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776745.821663:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776745.821665:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776745.821669:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821671:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776745.821673:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821675:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776745.821677:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776745.821679:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776745.821681:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776745.821683:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776745.821686:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776745.821688:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.821690:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776745.821692:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776745.821693:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.821695:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776745.821698:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776745.821702:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000800:00000010:2.0:1775776745.821703:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 02000000:00000001:0.0:1775776745.821705:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776745.821707:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776745.821710:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.0:1775776745.821711:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776745.821713:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.0:1775776745.821715:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 02000000:00000001:0.0:1775776745.821716:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776745.821721:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000010:0.0:1775776745.821722:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000800:00000200:2.1:1775776745.821725:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1775776745.821727:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775776745.821732:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00100000:0.0:1775776745.821735:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:2.0:1775776745.821736:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776745.821743:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775776745.821744:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776745.821748:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776745.821753:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1775776745.821755:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776745.821758:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776745.821763:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776745.821770:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776745.821803:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821812:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776745.821815:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821822:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821824:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776745.821828:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776745.821832:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776745.821836:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776745.821838:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313133574 00000100:00000001:2.0:1775776745.821840:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313134838 00000100:00000001:2.0:1775776745.821842:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313133574 00000100:00000001:2.0:1775776745.821844:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313134838 00000100:00000001:2.0:1775776745.821846:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776745.821848:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.821849:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776745.821853:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776745/real 1775776745] req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776745.821860:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776745.821861:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776745.821863:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776745.821865:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776745.821866:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776745.821869:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776745.821872:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776745.821879:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821885:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776745.821886:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.821888:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.821889:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776745.821892:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821898:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776745.821901:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821908:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776745.821910:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.821912:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.821913:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.821914:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776745.821916:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776745.821918:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776745.821922:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776745.821923:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776745.821927:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776745.821935:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789440:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776745.821939:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776745.821942:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821948:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776745.821949:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.821951:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776745.821953:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776745.821955:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776745.821958:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789568:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776745.821961:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776745.821964:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776745.821965:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.821967:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776745.821971:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776745.821973:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776745.821977:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776745.821979:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776745.821982:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776745.821985:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099789568, portal 10 00000100:00000001:2.0:1775776745.821987:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776745.821988:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:2.0:1775776745.821993:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.821999:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776745.822001:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776745.822004:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776745.822006:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099789568, offset 0 00000400:00000200:2.0:1775776745.822010:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776745.822014:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.822018:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776745.822021:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776745.822026:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776745.822029:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775776745.822031:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776745.822033:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000040:0.0:1775776745.822038:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18471) 00000400:00000200:2.0:1775776745.822040:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4399:4399:256:4294967295] 192.168.201.139@tcp LPNI seq info [4399:4399:8:4294967295] 00000400:00000200:2.0:1775776745.822046:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.822053:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776745.822058:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776745.822062:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776745.822067:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776745.822070:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776745.822074:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776745.822076:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1775776745.822079:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776745.822082:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776745.822084:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776745.822088:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776745.822091:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776745.822093:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776745.822095:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776745.822098:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822105:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776745.822108:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822114:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.822115:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776745.822118:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776745.822121:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776745.822124:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776745.822126:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313133575 00000100:00000001:2.0:1775776745.822129:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313134838 00000100:00000001:2.0:1775776745.822131:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313133575 00000100:00000001:2.0:1775776745.822133:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313134838 00000100:00000001:2.0:1775776745.822134:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776745.822136:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.822137:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776745.822140:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776745/real 1775776745] req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776745.822147:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776745.822148:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776745.822150:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776745.822152:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776745.822153:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776745.822155:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776745.822158:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776745.822165:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822171:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776745.822172:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822174:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.822175:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776745.822178:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822184:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776745.822188:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822194:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776745.822196:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822197:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.822198:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822200:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776745.822201:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776745.822203:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776745.822206:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776745.822208:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776745.822211:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776745.822218:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789568:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776745.822222:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822224:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776745.822226:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dfb80 x1862033099789440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822232:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776745.822233:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776745.822235:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776745.822237:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776745.822238:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776745.822239:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776745.822241:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776745.822242:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775776745.822245:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776745.822246:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776745.822247:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dfb80. 00000100:00000001:2.0:1775776745.822249:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776745.822250:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822252:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776745.822255:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099789568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822261:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776745.822262:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776745.822264:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776745.822265:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776745.822267:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776745.822268:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776745.822269:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776745.822271:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:2.0:1775776745.822273:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776745.822274:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776745.822275:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:2.0:1775776745.822277:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776745.822278:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822280:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822284:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776745.822285:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.822287:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776745.822288:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776745.822290:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776745.822293:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822299:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776745.822301:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.822303:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776745.822305:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776745.822306:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776745.822309:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789696:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776745.822312:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776745.822314:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776745.822315:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.822317:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776745.822319:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776745.822321:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776745.822325:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776745.822327:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776745.822330:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776745.822333:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099789696, portal 4 00000100:00000001:2.0:1775776745.822334:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776745.822336:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:2.0:1775776745.822340:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822347:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776745.822349:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776745.822351:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776745.822353:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099789696, offset 0 00000400:00000200:2.0:1775776745.822356:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776745.822360:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.822363:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776745.822365:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776745.822368:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776745.822370:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776745.822372:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776745.822375:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4400:4400:256:4294967295] 192.168.201.139@tcp LPNI seq info [4400:4400:8:4294967295] 00000400:00000200:2.0:1775776745.822379:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.822385:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776745.822389:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776745.822392:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776745.822396:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776745.822398:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776745.822402:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776745.822404:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1775776745.822408:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776745.822410:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776745.822412:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776745.822415:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776745.822418:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776745.822419:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776745.822421:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776745.822424:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822431:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776745.822434:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822440:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.822442:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776745.822444:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776745.822447:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776745.822449:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776745.822451:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313133584 00000100:00000001:2.0:1775776745.822454:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313134838 00000100:00000001:2.0:1775776745.822456:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313133584 00000100:00000001:2.0:1775776745.822458:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313134838 00000100:00000001:2.0:1775776745.822459:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776745.822461:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.822463:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776745.822467:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776745/real 1775776745] req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776745.822475:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776745.822477:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776745.822478:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776745.822480:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776745.822482:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776745.822484:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776745.822486:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776745.822495:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822501:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776745.822503:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822505:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.822506:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776745.822509:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822515:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776745.822518:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822537:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776745.822540:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822542:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.822543:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822545:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776745.822546:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776745.822549:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776745.822552:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776745.822555:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776745.822559:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776745.822567:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789696:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776745.822570:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776745.822573:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822579:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776745.822581:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.822583:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776745.822584:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776745.822586:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776745.822589:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789824:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776745.822592:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776745.822594:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776745.822595:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776745.822596:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776745.822599:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776745.822601:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776745.822604:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776745.822607:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776745.822609:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776745.822612:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099789824, portal 4 00000100:00000001:2.0:1775776745.822614:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776745.822615:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:2.0:1775776745.822620:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822627:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776745.822629:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776745.822631:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776745.822634:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099789824, offset 0 00000400:00000200:2.0:1775776745.822637:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776745.822641:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.822644:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776745.822646:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776745.822649:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776745.822651:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776745.822653:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776745.822656:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4401:4401:256:4294967295] 192.168.201.139@tcp LPNI seq info [4401:4401:8:4294967295] 00000400:00000200:2.0:1775776745.822660:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776745.822666:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776745.822670:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776745.822674:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776745.822677:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776745.822679:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776745.822683:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776745.822686:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1775776745.822690:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776745.822691:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776745.822694:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776745.822697:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776745.822699:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776745.822701:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776745.822703:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776745.822706:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822713:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776745.822716:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776770 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822722:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.822724:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776745.822726:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776745.822728:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776745.822731:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776745.822733:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313133575 00000100:00000001:2.0:1775776745.822735:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313134838 00000100:00000001:2.0:1775776745.822737:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313133575 00000100:00000001:2.0:1775776745.822739:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313134838 00000100:00000001:2.0:1775776745.822740:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776745.822742:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.822744:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776745.822747:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776745/real 1775776745] req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776745.822754:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776745.822756:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776745.822757:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776745.822759:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776745.822761:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776745.822762:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776745.822765:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776745.822773:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822797:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776745.822798:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822800:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.822801:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776745.822805:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822811:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776745.822815:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822821:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776745.822824:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822825:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776745.822826:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822828:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776745.822829:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776745.822832:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776745.822834:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776745.822836:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776745.822839:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776745.822846:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099789824:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776745.822850:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822851:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776745.822854:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099789696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822860:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776745.822862:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776745.822863:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776745.822866:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776745.822867:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776745.822868:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776745.822870:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776745.822872:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c000. 02000000:00000001:2.0:1775776745.822874:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776745.822875:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776745.822876:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:2.0:1775776745.822879:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776745.822880:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822881:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776745.822884:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099789824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776770 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776745.822896:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776745.822897:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776745.822899:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776745.822901:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776745.822902:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776745.822903:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776745.822905:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776745.822906:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759ec00. 02000000:00000001:2.0:1775776745.822907:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776745.822909:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776745.822910:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:2.0:1775776745.822911:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776745.822913:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822914:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776745.822917:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776745.822918:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776745.822920:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776745.822921:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776746.524152:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776746.524160:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776746.524161:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776746.524167:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776746.524171:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776746.524175:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776746.524177:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776746.524180:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776746.524183:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776746.587899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776746.587904:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776746.587908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776746.587909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776746.587911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776746.587912:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776746.587914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776746.587915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776746.588015:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776746.588019:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776746.588021:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776746.588100:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776746.588104:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776746.588105:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776746.588109:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776746.588112:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776746.588116:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776746.588118:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776746.588120:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776746.588123:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776747.099907:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313134838, at 4313134848 00000100:00000010:0.0:1775776747.099923:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000040:1.0:1775776747.100036:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313134838, at 4313134848 00000100:00000010:1.0:1775776747.100044:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:3.0:1775776747.548302:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776747.548311:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776747.548313:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776747.548318:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776747.548322:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776747.548326:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776747.548328:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776747.548330:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776747.548338:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776747.611907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776747.611910:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776747.611915:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1775776747.611915:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1775776747.611919:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1775776747.611919:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776747.611924:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776747.611925:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776747.611929:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1775776747.611930:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776747.611934:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776747.611935:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776747.611938:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776747.611939:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776747.611942:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776747.611944:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776747.611947:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776747.611948:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776747.611951:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776747.611956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776747.611958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776747.611961:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776747.611962:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776747.612070:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776747.612074:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776747.612076:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776748.571858:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776748.571865:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776748.571867:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776748.571872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776748.571875:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776748.571879:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776748.571881:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776748.571884:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776748.571886:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776748.635894:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00010000:00000001:2.0:1775776748.635897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776748.635901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776748.635901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776748.635905:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776748.635909:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776748.635910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776748.635911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776748.635917:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1775776748.635918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776748.635921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776748.635921:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776748.635925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1775776748.635926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776748.635928:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776748.635929:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776748.635932:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776748.637032:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776748.637036:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776748.637040:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776749.595903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776749.595910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776749.595912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776749.595919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776749.595923:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776749.595928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776749.595930:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776749.595932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776749.595935:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776749.661024:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776749.661039:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776749.661041:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776749.661046:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776749.661049:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776749.661053:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776749.661056:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776749.661058:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776749.661061:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776749.661650:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776749.661653:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776749.661658:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776749.661660:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776749.661663:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776749.661664:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776749.661667:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776749.661668:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776749.661958:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776749.661961:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776749.661964:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776750.619887:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776750.619893:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776750.619895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776750.619900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776750.619903:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776750.619907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776750.619910:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776750.619913:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776750.619915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776750.683891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776750.683898:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776750.683900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1775776750.683905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00010000:00000001:2.0:1775776750.683908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776750.683910:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776750.683914:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1775776750.683916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1775776750.683917:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776750.683921:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776750.683923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683929:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776750.683932:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776750.683936:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683939:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776750.683942:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683944:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683946:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776750.683952:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776750.683953:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776750.683955:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683956:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776750.683958:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683959:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683961:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683964:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776750.683966:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776750.683967:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776750.683968:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683970:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776750.683971:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683973:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776750.683974:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776750.684250:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776750.684253:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776750.684256:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776750.940453:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776750.940473:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776750.940476:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776750.940483:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776750.940488:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776750.940498:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18466 00000400:00000200:1.0:1775776750.940510:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776750.940518:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776750.940521:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776750.940525:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940529:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.940531:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940534:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776750.940539:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776750.940541:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776750.940545:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776750.940548:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776750.940551:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776750.940553:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776750.940557:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940559:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.940561:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940563:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776750.940565:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776750.940569:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940572:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776750.940575:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776750.940580:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776750.940583:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.940586:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776750.940589:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776750.940601:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775776750.940606:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776750.940611:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776750.940613:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.940617:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776750.940620:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776750.940625:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776750.940638:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776750.940646:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776750.940657:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776750.940668:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776750.940674:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776750.940680:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.940757:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776750.940763:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776750.940766:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776750.940773:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776750.940848:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776750.940852:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776750.940856:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776750.940857:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776750.940859:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776750.940862:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776750.940865:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18466 00000400:00000200:1.0:1775776750.940870:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776750.940873:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776750.940875:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776750.940878:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940880:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.940882:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940884:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776750.940887:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776750.940888:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776750.940890:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776750.940892:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776750.940894:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776750.940895:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776750.940899:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940900:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.940902:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940904:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776750.940906:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776750.940908:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776750.940911:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776750.940913:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776750.940919:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776750.940922:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.940923:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776750.940928:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776750.940931:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.940933:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776750.940939:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775776750.940941:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776750.940944:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776750.940945:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.940947:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776750.940948:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776750.940950:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776750.940955:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776750.940959:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776750.940964:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776750.940972:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776750.940974:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776750.940978:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776750.940986:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776750.940989:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776750.940992:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776750.940993:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776750.940995:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776750.940998:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776750.941000:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18466 00000400:00000200:1.0:1775776750.941004:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776750.941007:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776750.941010:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776750.941089:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776750.941091:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.941095:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776750.941098:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776750.941101:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776750.941105:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099789952:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776750.941109:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776750.941114:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776750.941117:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.941120:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776750.941127:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776750.941129:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776750.941136:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776750.941141:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776750.941150:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776750.941644:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099789952, portal 25 00000100:00000001:2.0:1775776750.941648:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776750.941650:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287611776 : -114841421939840 : ffff978d65d6fb80) 00000100:00000040:2.0:1775776750.941658:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.941667:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776750.941671:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776750.941675:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776750.941680:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099789952, offset 0 00000400:00000200:2.0:1775776750.941687:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776750.941696:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.941703:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776750.941709:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776750.941714:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776750.941719:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776750.941722:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776750.941726:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4402:4402:256:4294967295] 192.168.201.139@tcp LPNI seq info [4402:4402:8:4294967295] 00000400:00000200:2.0:1775776750.941731:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.941744:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776750.941751:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.941758:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776750.941766:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776750.941771:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1775776750.941818:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941822:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.941824:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941827:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776750.941829:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776750.941832:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776750.941834:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776750.941836:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776750.941838:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776750.941841:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776750.941844:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941846:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.941848:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941850:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776750.941852:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776750.941855:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941857:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776750.941859:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776750.941863:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776750.941865:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.941868:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776750.941872:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776750.941874:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.941876:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776750.941880:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775776750.941883:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776750.941885:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776750.941887:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.941889:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776750.941890:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776750.941893:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776750.941898:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776750.941902:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776750.941907:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776750.941915:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776750.941917:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776750.941922:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776750.941931:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776750.941933:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776750.941937:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776750.941939:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776750.941940:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776750.941944:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776750.941946:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18466 00000400:00000200:1.0:1775776750.941951:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776750.941954:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776750.941957:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776750.941959:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941962:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.941964:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941966:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776750.941968:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776750.941969:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776750.941971:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776750.941973:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776750.941975:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776750.941977:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776750.941980:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941982:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.941983:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941985:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776750.941987:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776750.941990:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776750.941992:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776750.941995:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776750.941998:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776750.942000:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.942002:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776750.942005:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776750.942007:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.942008:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776750.942012:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000800:00000200:2.1:1775776750.942550:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776750.942557:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776750.942563:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.942567:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776750.942575:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776750.942581:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776750.942588:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776750.942591:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776750.942597:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776750.942601:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.942612:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1775776750.942637:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776750.942641:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776750.942642:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.942644:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776750.942646:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776750.942648:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776750.942653:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776750.942657:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776750.942663:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776750.942671:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776750.942674:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776750.942678:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776750.942685:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776750.942688:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776750.942691:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776750.942693:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776750.942694:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776750.942697:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776750.942699:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18466 00000400:00000200:1.0:1775776750.942704:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776750.942707:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776750.942709:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776750.942712:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.942714:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.942716:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.942718:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776750.942720:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776750.942721:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776750.942723:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776750.942725:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776750.942727:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776750.942729:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776750.942731:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.942733:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776750.942735:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776750.942736:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776750.942739:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776750.942741:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776750.942743:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776750.942745:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776750.942748:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776750.942750:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.942752:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776750.942754:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776750.942756:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.942758:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776750.942761:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775776750.942764:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776750.942766:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776750.942767:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776750.942769:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776750.942770:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776750.942771:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776750.942775:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776750.942795:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776750.942800:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776750.942808:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776750.942810:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776750.942812:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776750.942819:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776750.942824:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18476) 00000100:00000040:2.0:1775776750.942873:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.942881:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.942883:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776750.942907:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776750.942912:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776750.942915:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313139960 00000100:00000010:2.0:1775776750.942921:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b5a0. 00000100:00000001:2.0:1775776750.942930:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776750.943327:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313139960 00000100:00000010:2.0:1775776750.943333:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775776750.943337:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776750.943341:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776750.943345:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.943348:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776750.943354:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776750/real 1775776750] req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776750.943368:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776750.943370:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776750.943373:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776750.943376:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776750.943378:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776750.943381:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776750.943385:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776750.943394:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.943401:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776750.943403:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.943405:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.943408:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776750.943411:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.943417:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776750.943421:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.943429:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776750.943432:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.943835:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.943837:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.943840:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776750.943842:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776750.943846:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776750.943851:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776750.943854:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776750.943859:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776750.943867:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099789952:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776750.943872:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.943875:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.943877:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6fb80 x1862033099789952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.943883:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776750.943886:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776750.943889:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776750.943893:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776750.943895:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776750.943896:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776750.943898:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776750.943901:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38000. 02000000:00000001:2.0:1775776750.943905:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776750.943907:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776750.943908:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6fb80. 00000100:00000001:2.0:1775776750.943912:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776750.943913:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.943915:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.943925:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776750.943927:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.943930:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776750.943932:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776750.943934:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776750.943938:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.943945:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776750.943947:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.943950:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776750.943953:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776750.943955:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776750.943959:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790080:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776750.943962:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776750.943965:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776750.943968:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.943970:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776750.943975:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776750.943977:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776750.943982:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776750.943986:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776750.943991:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776750.943997:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099790080, portal 10 00000100:00000001:2.0:1775776750.943999:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776750.944000:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287603712 : -114841421947904 : ffff978d65d6dc00) 00000100:00000040:2.0:1775776750.944007:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944134:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776750.944137:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776750.944140:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776750.944145:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099790080, offset 0 00000400:00000200:2.0:1775776750.944150:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776750.944275:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.944282:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776750.944287:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776750.944291:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776750.944295:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776750.944298:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776750.944301:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4403:4403:256:4294967295] 192.168.201.139@tcp LPNI seq info [4403:4403:8:4294967295] 00000400:00000200:2.0:1775776750.944305:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.944316:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776750.944321:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.944327:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776750.944334:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776750.944337:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776750.944343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776750.944346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776750.944350:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.944352:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776750.944357:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776750.944361:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776750.944365:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776750.944367:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776750.944370:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776750.944374:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944383:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.944386:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944393:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.944395:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776750.944399:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776750.944404:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776750.944407:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776750.944409:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313138696 00000100:00000001:2.0:1775776750.944412:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313139960 00000100:00000001:2.0:1775776750.944414:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313138696 00000100:00000001:2.0:1775776750.944416:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313139960 00000100:00000001:2.0:1775776750.944418:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776750.944420:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.944422:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776750.944426:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776750/real 1775776750] req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776750.944446:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776750.944448:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776750.944450:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776750.944453:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776750.944454:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776750.944457:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776750.944460:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776750.944468:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944474:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776750.944475:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.944477:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.944478:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776750.944481:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944487:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776750.944490:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944497:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776750.944499:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.944501:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.944502:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.944504:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776750.944506:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776750.944509:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776750.944512:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776750.944514:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776750.944518:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776750.944524:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790080:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776750.944528:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776750.944531:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944537:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776750.944538:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.944540:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776750.944542:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776750.944544:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776750.944546:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790208:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776750.944550:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776750.944552:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776750.944553:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.944555:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776750.944559:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776750.944561:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776750.944564:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776750.944566:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776750.944569:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776750.944572:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099790208, portal 10 00000100:00000001:2.0:1775776750.944574:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776750.944575:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287597440 : -114841421954176 : ffff978d65d6c380) 00000100:00000040:2.0:1775776750.944581:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944587:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776750.944590:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776750.944592:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776750.944595:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099790208, offset 0 00000400:00000200:2.0:1775776750.944599:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776750.944603:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.944606:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776750.944609:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776750.944611:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776750.944614:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776750.944616:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776750.944619:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4404:4404:256:4294967295] 192.168.201.139@tcp LPNI seq info [4404:4404:8:4294967295] 00000400:00000200:2.0:1775776750.944622:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.944628:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776750.944632:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.944635:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776750.944638:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776750.944641:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776750.944645:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776750.944648:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776750.944652:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.944654:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776750.944657:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776750.944660:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776750.944663:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776750.944665:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776750.944667:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776750.944670:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944677:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.944681:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944688:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.944689:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776750.944692:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776750.944695:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776750.944698:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776750.944700:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313138697 00000100:00000001:2.0:1775776750.944702:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313139960 00000100:00000001:2.0:1775776750.944704:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313138697 00000100:00000001:2.0:1775776750.944706:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313139960 00000100:00000001:2.0:1775776750.944708:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776750.944710:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.944712:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776750.944715:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776750/real 1775776750] req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776750.944722:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776750.944723:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776750.944725:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776750.944727:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776750.944728:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776750.944730:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776750.944733:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776750.944740:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944745:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776750.944746:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.944748:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.944749:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776750.944752:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944757:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776750.944760:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944766:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776750.944768:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.944769:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.944770:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.944772:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776750.944773:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776750.944775:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776750.944803:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776750.944806:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776750.944810:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776750.944818:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790208:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776750.944822:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776750.944826:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944832:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776750.944833:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.944836:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776750.944838:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776750.944840:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776750.944843:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790336:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776750.944846:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776750.944848:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776750.944850:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.944851:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776750.944855:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776750.944857:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776750.944860:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776750.944863:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776750.944867:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776750.944870:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099790336, portal 4 00000100:00000001:2.0:1775776750.944872:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776750.944873:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287598336 : -114841421953280 : ffff978d65d6c700) 00000100:00000040:2.0:1775776750.944879:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944885:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776750.944888:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776750.944890:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776750.944893:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099790336, offset 0 00000400:00000200:2.0:1775776750.944897:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776750.944900:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.944904:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776750.944906:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776750.944909:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776750.944911:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776750.944914:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776750.944917:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4405:4405:256:4294967295] 192.168.201.139@tcp LPNI seq info [4405:4405:8:4294967295] 00000400:00000200:2.0:1775776750.944920:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.944927:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776750.944931:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.944934:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776750.944938:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776750.944940:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776750.944944:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776750.944946:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776750.944950:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.944953:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776750.944956:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776750.944959:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776750.944962:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776750.944963:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776750.944966:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776750.944969:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944976:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.944979:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.944986:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.944987:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776750.944990:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776750.944993:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776750.944995:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776750.944997:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313138707 00000100:00000001:2.0:1775776750.945000:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313139960 00000100:00000001:2.0:1775776750.945002:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313138707 00000100:00000001:2.0:1775776750.945003:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313139960 00000100:00000001:2.0:1775776750.945005:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776750.945007:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.945009:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776750.945012:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776750/real 1775776750] req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776750.945154:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776750.945155:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776750.945164:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776750.945167:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776750.945169:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776750.945172:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776750.945176:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776750.945184:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945317:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776750.945320:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945322:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.945324:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776750.945328:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945335:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776750.945339:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945345:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776750.945348:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945350:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.945351:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945353:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776750.945354:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776750.945357:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776750.945360:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776750.945363:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776750.945366:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776750.945374:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790336:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776750.945378:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776750.945382:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945388:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776750.945390:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.945392:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776750.945394:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776750.945396:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776750.945399:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790464:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776750.945402:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776750.945404:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776750.945406:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776750.945408:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776750.945412:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776750.945414:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776750.945417:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776750.945420:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776750.945424:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776750.945426:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099790464, portal 4 00000100:00000001:2.0:1775776750.945428:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776750.945430:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609088 : -114841421942528 : ffff978d65d6f100) 00000100:00000040:2.0:1775776750.945453:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945459:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776750.945461:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776750.945464:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776750.945466:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099790464, offset 0 00000400:00000200:2.0:1775776750.945471:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776750.945475:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.945478:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776750.945481:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776750.945484:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776750.945486:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776750.945488:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776750.945492:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4406:4406:256:4294967295] 192.168.201.139@tcp LPNI seq info [4406:4406:8:4294967295] 00000400:00000200:2.0:1775776750.945496:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776750.945503:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776750.945508:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.945511:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776750.945515:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776750.945518:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776750.945522:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776750.945525:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776750.945528:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776750.945530:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776750.945533:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776750.945536:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776750.945539:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776750.945540:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776750.945543:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776750.945546:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945554:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.945557:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776775 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945563:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.945565:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776750.945567:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776750.945570:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776750.945574:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776750.945576:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313138698 00000100:00000001:2.0:1775776750.945578:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313139960 00000100:00000001:2.0:1775776750.945580:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313138698 00000100:00000001:2.0:1775776750.945582:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313139960 00000100:00000001:2.0:1775776750.945584:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776750.945587:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.945589:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776750.945592:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776750/real 1775776750] req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776750.945601:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776750.945602:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776750.945604:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776750.945606:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776750.945607:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776750.945609:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776750.945611:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776750.945619:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945624:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776750.945625:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945627:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.945628:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776750.945631:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945636:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776750.945639:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945646:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776750.945648:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945649:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776750.945651:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945652:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776750.945654:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776750.945656:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776750.945659:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776750.945661:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776750.945664:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776750.945671:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790464:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776750.945674:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945676:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.945679:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6dc00 x1862033099790080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945684:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776750.945687:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776750.945689:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776750.945691:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776750.945692:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776750.945694:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776750.945696:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776750.945698:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3ac00. 02000000:00000001:2.0:1775776750.945701:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776750.945702:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776750.945704:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6dc00. 00000100:00000001:2.0:1775776750.945706:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776750.945708:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945709:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.945712:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c380 x1862033099790208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945717:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776750.945719:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776750.945720:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776750.945722:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776750.945723:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776750.945724:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776750.945726:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776750.945727:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39c00. 02000000:00000001:2.0:1775776750.945730:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776750.945730:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776750.945732:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c380. 00000100:00000001:2.0:1775776750.945734:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776750.945735:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945736:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.945739:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c700 x1862033099790336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945744:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776750.945745:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776750.945746:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776750.945748:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776750.945749:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776750.945750:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776750.945752:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776750.945753:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a000. 02000000:00000001:2.0:1775776750.945755:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776750.945756:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776750.945757:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c700. 00000100:00000001:2.0:1775776750.945759:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776750.945761:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945762:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776750.945765:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f100 x1862033099790464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776775 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776750.945771:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776750.945772:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776750.945774:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776750.945775:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776750.945777:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776750.945982:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776750.945985:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776750.945987:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b000. 02000000:00000001:2.0:1775776750.945990:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776750.945991:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776750.945992:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f100. 00000100:00000001:2.0:1775776750.945995:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776750.945997:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.945999:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776750.946005:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776750.946006:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776750.946008:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776750.946010:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776751.643991:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776751.643997:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776751.643998:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776751.644003:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776751.644006:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776751.644009:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776751.644011:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776751.644013:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776751.644015:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776751.707985:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776751.707992:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776751.708000:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776751.708001:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776751.708004:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776751.708005:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776751.708009:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776751.708010:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776751.708057:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776751.708080:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776751.708082:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776751.708087:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776751.708091:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776751.708094:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776751.708097:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776751.708099:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776751.708102:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776751.708700:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776751.708705:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776751.708709:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776752.219946:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313139960, at 4313139968 00000100:00000010:0.0:1775776752.219960:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000040:1.0:1775776752.220008:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313139960, at 4313139968 00000100:00000010:1.0:1775776752.220024:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775776752.668400:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776752.668416:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776752.668418:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776752.668425:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776752.668429:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776752.668434:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776752.668437:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776752.668440:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776752.668443:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776752.731953:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776752.731959:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776752.731963:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776752.731964:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776752.731967:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776752.731968:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776752.731971:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776752.731971:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776752.732173:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776752.732177:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776752.732178:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776752.732182:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776752.732185:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776752.732188:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776752.732190:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776752.732192:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776752.732195:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776752.732238:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776752.732240:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776752.732243:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776753.691930:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776753.691939:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776753.691941:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776753.691947:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776753.691951:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776753.691956:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776753.691959:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776753.691961:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776753.691965:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776753.756074:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776753.756082:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776753.756084:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776753.756090:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776753.756095:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776753.756100:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776753.756103:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776753.756105:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776753.756108:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776753.756404:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776753.756408:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776753.756414:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776753.756415:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776753.756419:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776753.756421:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776753.756424:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776753.756426:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776753.756532:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776753.756535:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776753.756539:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776754.715937:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776754.715946:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776754.715948:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776754.715955:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776754.715959:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776754.715963:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776754.715966:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776754.715969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776754.715972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776754.781711:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776754.781717:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776754.781723:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776754.781724:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776754.781728:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776754.781730:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776754.781733:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776754.781735:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776754.812495:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776754.812582:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776754.812589:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776754.812729:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776754.812735:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776754.812737:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776754.812742:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776754.812747:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776754.812835:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776754.812840:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776754.812843:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776754.812846:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776755.740653:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776755.740661:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776755.740663:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776755.740668:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776755.740672:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776755.740677:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776755.740680:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776755.740683:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776755.740685:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776755.804028:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776755.804035:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776755.804670:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776755.804673:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776755.804679:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776755.804680:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776755.804683:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776755.804684:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776755.804815:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776755.804828:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776755.804832:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776755.867943:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776755.867951:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776755.867953:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776755.867960:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776755.867965:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776755.867970:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776755.867973:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776755.867977:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776755.867980:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776756.059923:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776756.059934:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776756.059937:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776756.059941:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776756.059945:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776756.059952:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18471 00000400:00000200:0.0:1775776756.059962:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776756.060871:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776756.060877:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776756.060881:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.060884:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.060887:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.060890:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776756.060894:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776756.060897:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776756.060901:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776756.060904:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776756.060907:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776756.060909:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776756.060913:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.060915:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.060916:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.060918:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776756.060922:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776756.060925:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776756.060928:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776756.060931:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776756.060936:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776756.060939:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.060941:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776756.060944:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776756.060953:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776756.060957:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776756.060961:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776756.060963:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.060966:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776756.060968:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776756.060973:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776756.060980:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776756.060987:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776756.060996:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776756.061007:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776756.061012:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776756.061017:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.061800:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776756.061804:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776756.061807:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776756.061822:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.061831:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776756.061833:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.061836:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776756.061838:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776756.061840:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776756.061843:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099790592:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776756.061847:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776756.061851:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776756.061853:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.061855:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776756.061861:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776756.061863:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776756.061870:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776756.061875:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776756.061881:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776756.061888:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099790592, portal 25 00000100:00000001:2.0:1775776756.061890:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776756.061891:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:2.0:1775776756.061897:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.061903:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776756.061906:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776756.061909:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776756.061913:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099790592, offset 0 00000400:00000200:2.0:1775776756.061920:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776756.061928:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.061935:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776756.061939:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776756.061942:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776756.061946:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776756.061949:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776756.061953:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4407:4407:256:4294967295] 192.168.201.139@tcp LPNI seq info [4407:4407:8:4294967295] 00000100:00000001:0.0:1775776756.062366:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776756.062370:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776756.062374:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776756.062375:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776756.062378:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776756.062381:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776756.062384:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18471 00000400:00000200:0.0:1775776756.062396:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776756.062399:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776756.062401:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776756.062404:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.062407:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.062409:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.062411:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776756.062414:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776756.062415:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776756.062417:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776756.062419:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776756.062420:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776756.062422:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776756.062425:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.062427:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.062429:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.062431:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776756.062433:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776756.062435:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776756.062437:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776756.062439:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776756.062444:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776756.062447:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.062448:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776756.062452:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776756.062455:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.062457:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776756.062462:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776756.062464:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776756.062467:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776756.062468:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.062470:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776756.062472:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776756.062475:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776756.062479:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776756.062483:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776756.062488:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776756.062496:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776756.062500:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776756.062503:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776756.062512:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776756.062515:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776756.062518:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776756.062519:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776756.062521:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776756.062524:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776756.062526:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18471 00000400:00000200:0.0:1775776756.062530:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776756.062534:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776756.062536:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776756.062538:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.062540:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776756.062916:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.062931:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776756.062937:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.062944:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776756.062951:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776756.062955:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776756.062960:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776756.062963:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776756.062967:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.062969:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776756.062974:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776756.062979:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776756.062983:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776756.062986:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776756.062990:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776756.062995:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.063002:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.063005:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.063012:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.063013:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776756.063018:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776756.063022:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776756.063024:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313145080 00000100:00000010:2.0:1775776756.063029:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b1e0. 00000100:00000001:2.0:1775776756.063034:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776756.063042:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313145080 00000100:00000010:2.0:1775776756.063045:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bba0. 00000100:00000001:2.0:1775776756.063048:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776756.063051:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776756.063054:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.063056:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776756.063060:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776756/real 1775776756] req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776756.063069:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776756.063071:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776756.063073:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776756.063076:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776756.063077:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776756.063079:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776756.063082:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776756.063090:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.063101:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776756.063102:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.063104:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.063106:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776756.063109:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.063116:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776756.063120:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.063128:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776756.063130:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.063132:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.063133:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.063135:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776756.063137:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776756.063139:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776756.063143:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776756.063145:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776756.063150:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776756.063157:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099790592:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776756.063161:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.063163:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.063166:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099790592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.063172:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776756.063175:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776756.063178:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776756.063180:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776756.063182:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776756.063185:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776756.063187:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776756.063189:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759cc00. 02000000:00000001:2.0:1775776756.063193:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776756.063195:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776756.063197:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000100:00000001:2.0:1775776756.063199:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776756.063200:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.063202:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.063209:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776756.063210:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.063213:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776756.063214:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776756.063216:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776756.063219:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.063225:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776756.063226:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.063229:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776756.063230:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776756.063232:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776756.063235:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790720:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776756.063239:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776756.063241:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776756.063243:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.063245:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776756.063248:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776756.063250:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776756.063254:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776756.063257:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776756.063261:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776756.063265:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099790720, portal 10 00000100:00000040:0.0:1775776756.064482:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.064486:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776756.064490:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776756.064492:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776756.064495:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776756.064498:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776756.064500:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776756.064503:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776756.064507:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.064509:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.064512:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.064514:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776756.064516:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776756.064519:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776756.064522:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776756.064524:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776756.064530:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776756.064532:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.064535:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776756.064538:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776756.064541:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.064543:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776756.064551:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776756.064796:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776756.064800:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776756.064801:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.064804:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776756.064806:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776756.065079:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776756.065086:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776756.065091:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776756.065098:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776756.065208:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776756.065212:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776756.065216:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776756.065227:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776756.065230:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776756.065234:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776756.065235:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776756.065237:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776756.065239:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776756.065242:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18471 00000400:00000200:0.0:1775776756.065249:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776756.065253:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776756.065255:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776756.065258:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.065260:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.065262:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.065263:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776756.065265:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776756.065267:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776756.065269:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776756.065271:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776756.065273:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776756.065275:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776756.065277:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.065278:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.065280:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.065281:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776756.065283:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776756.065285:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776756.065287:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776756.065288:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776756.065291:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776756.065293:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.065294:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776756.065296:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776756.065298:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.065300:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776756.065303:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776756.065305:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776756.065306:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776756.065307:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.065309:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776756.065310:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776756.065311:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776756.065315:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776756.065318:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776756.065321:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776756.065327:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776756.065329:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776756.065333:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776756.065340:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776756.065342:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776756.065355:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776756.065356:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776756.065357:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776756.065360:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776756.065362:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18471 00000400:00000200:0.0:1775776756.065365:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776756.065367:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776756.065369:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776756.065371:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.065373:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.065374:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.065376:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776756.065377:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776756.065378:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776756.065379:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776756.065381:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776756.065383:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776756.065384:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:2.0:1775776756.066617:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776756.066622:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000100:00000040:2.0:1775776756.066634:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.066643:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776756.066646:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776756.066653:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776756.066658:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099790720, offset 0 00000400:00000200:2.0:1775776756.066665:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776756.066674:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.066680:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776756.066685:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776756.066689:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776756.066692:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776756.066695:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776756.066699:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4408:4408:256:4294967295] 192.168.201.139@tcp LPNI seq info [4408:4408:8:4294967295] 00000400:00000200:2.0:1775776756.066704:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.066713:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776756.066719:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.066724:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776756.066730:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776756.066734:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776756.066739:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776756.066742:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776756.066747:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.066749:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776756.066755:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776756.066759:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776756.066764:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776756.066766:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776756.066771:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776756.066775:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.066806:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.066810:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.066817:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.066819:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776756.066824:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776756.066828:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776756.066832:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776756.066835:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313143817 00000100:00000001:2.0:1775776756.066839:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313145080 00000100:00000001:2.0:1775776756.066841:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313143817 00000100:00000001:2.0:1775776756.066843:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313145080 00000100:00000001:2.0:1775776756.066845:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776756.066848:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.066851:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776756.066855:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776756/real 1775776756] req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776756.066866:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776756.066868:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776756.066870:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776756.066873:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776756.066875:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776756.066878:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776756.066881:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776756.066888:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.066894:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776756.066896:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.066898:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.066900:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776756.066903:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.066909:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776756.066913:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.066920:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776756.066923:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.066924:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.066926:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.066928:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776756.066931:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776756.066934:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776756.066937:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776756.066940:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776756.066944:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776756.066951:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790720:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776756.066955:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.066957:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.066960:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099790720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.066967:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776756.066969:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776756.066972:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776756.066975:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776756.066977:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776756.066979:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776756.066981:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776756.066984:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e400. 02000000:00000001:2.0:1775776756.066988:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776756.066989:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776756.066991:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:2.0:1775776756.066994:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776756.066996:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.066998:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067004:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776756.067006:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.067008:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776756.067010:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776756.067012:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776756.067016:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067023:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776756.067025:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.067027:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776756.067030:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776756.067033:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776756.067036:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790848:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776756.067039:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776756.067043:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776756.067045:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.067048:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776756.067051:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776756.067054:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776756.067058:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776756.067061:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776756.067065:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776756.067070:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099790848, portal 10 00000100:00000001:2.0:1775776756.067072:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776756.067073:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748096 : -114841967803520 : ffff978d454dc380) 00000100:00000040:2.0:1775776756.067078:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067085:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776756.067086:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776756.067089:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776756.067091:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099790848, offset 0 00000400:00000200:2.0:1775776756.067095:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776756.067099:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.067103:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776756.067106:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776756.067108:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776756.067110:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776756.067112:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776756.067115:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4409:4409:256:4294967295] 192.168.201.139@tcp LPNI seq info [4409:4409:8:4294967295] 00000400:00000200:2.0:1775776756.067119:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.067125:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776756.067130:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.067133:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776756.067137:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776756.067139:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776756.067143:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776756.067146:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776756.067149:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.067151:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776756.067153:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776756.067156:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776756.067159:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776756.067160:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776756.067163:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776756.067166:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067172:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.067175:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067185:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.067186:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776756.067189:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776756.067191:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776756.067194:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776756.067196:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313143818 00000100:00000001:2.0:1775776756.067198:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313145080 00000100:00000001:2.0:1775776756.067200:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313143818 00000100:00000001:2.0:1775776756.067202:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313145080 00000100:00000001:2.0:1775776756.067204:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776756.067205:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.067207:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776756.067210:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776756/real 1775776756] req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776756.067216:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776756.067218:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776756.067219:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776756.067221:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776756.067222:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776756.067224:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776756.067227:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776756.067233:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067239:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776756.067240:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067241:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.067242:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776756.067246:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067252:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776756.067255:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067260:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776756.067263:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067264:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.067265:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067267:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776756.067268:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776756.067270:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776756.067273:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776756.067275:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776756.067278:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776756.067285:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790848:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776756.067288:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776756.067292:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067297:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776756.067298:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.067300:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776756.067302:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776756.067304:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776756.067306:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790976:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776756.067309:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776756.067311:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776756.067313:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.067315:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776756.067318:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776756.067319:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776756.067323:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776756.067325:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776756.067328:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776756.067330:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099790976, portal 4 00000100:00000001:2.0:1775776756.067332:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776756.067333:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741750784 : -114841967800832 : ffff978d454dce00) 00000100:00000040:2.0:1775776756.067337:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067343:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776756.067361:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776756.067363:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776756.067365:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099790976, offset 0 00000400:00000200:2.0:1775776756.067369:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776756.067373:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.067376:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776756.067378:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776756.067380:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776756.067382:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776756.067385:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776756.067387:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4410:4410:256:4294967295] 192.168.201.139@tcp LPNI seq info [4410:4410:8:4294967295] 00000400:00000200:2.0:1775776756.067391:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.067406:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776756.067410:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.067414:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776756.067417:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776756.067420:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776756.067423:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776756.067425:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776756.067429:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.067431:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776756.067433:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776756.067436:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776756.067439:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776756.067440:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776756.067442:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776756.067446:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067453:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.067456:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067462:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.067463:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776756.067466:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776756.067469:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776756.067472:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776756.067473:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313143827 00000100:00000001:2.0:1775776756.067476:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313145080 00000100:00000001:2.0:1775776756.067478:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313143827 00000100:00000001:2.0:1775776756.067480:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313145080 00000100:00000001:2.0:1775776756.067481:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776756.067483:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.067485:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776756.067488:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776756/real 1775776756] req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776756.067495:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776756.067496:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776756.067498:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776756.067500:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776756.067502:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776756.067504:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776756.067507:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776756.067515:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067521:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776756.067522:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067524:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.067525:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776756.067528:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067534:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776756.067537:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067544:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776756.067546:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067547:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.067549:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067550:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776756.067552:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776756.067554:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776756.067557:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776756.067559:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776756.067562:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776756.067569:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099790976:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776756.067572:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067574:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.067577:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc380 x1862033099790848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067582:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776756.067584:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776756.067585:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776756.067588:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776756.067589:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776756.067591:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776756.067592:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776756.067594:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:2.0:1775776756.067596:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776756.067597:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776756.067599:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc380. 00000100:00000001:2.0:1775776756.067601:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776756.067602:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067604:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.067607:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dce00 x1862033099790976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.067613:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776756.067614:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776756.067615:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776756.067617:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776756.067618:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776756.067619:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776756.067620:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776756.067622:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f800. 02000000:00000001:2.0:1775776756.067624:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776756.067625:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776756.067626:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dce00. 00000100:00000001:2.0:1775776756.067628:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776756.067630:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067631:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.067635:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776756.067636:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.067638:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776756.067639:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776756.069421:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.069428:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776756.069431:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776756.069433:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776756.069437:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776756.069441:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776756.069444:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776756.069448:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776756.069464:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776756.069467:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.069470:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776756.069473:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776756.069476:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.069478:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776756.069486:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776756.069489:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776756.069493:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776756.069494:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776756.069502:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776756.069504:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776756.069508:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776756.069515:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776756.069521:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776756.069528:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776756.069538:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776756.069542:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776756.069546:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776756.069567:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776756.069574:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18481) 00000100:00000001:2.0:1775776756.070912:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776756.070917:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776756.070919:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776756.070927:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.070938:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776756.070939:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.070943:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776756.070946:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776756.070948:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776756.070951:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791104:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776756.070955:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776756.070958:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776756.070959:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776756.070962:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776756.070968:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776756.070970:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776756.070977:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776756.070981:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776756.070987:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776756.070994:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099791104, portal 4 00000100:00000001:2.0:1775776756.070996:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776756.070998:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748992 : -114841967802624 : ffff978d454dc700) 00000100:00000040:2.0:1775776756.071005:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.071013:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776756.071016:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776756.071018:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776756.071022:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099791104, offset 0 00000400:00000200:2.0:1775776756.071028:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776756.071037:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.071044:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776756.071049:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776756.071053:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776756.071057:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776756.071059:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776756.071062:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4411:4411:256:4294967295] 192.168.201.139@tcp LPNI seq info [4411:4411:8:4294967295] 00000400:00000200:2.0:1775776756.071066:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776756.071076:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776756.071082:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.071088:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776756.071093:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776756.071096:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776756.071101:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776756.071104:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776756.071108:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776756.071110:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776756.071116:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776756.071120:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776756.071124:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776756.071126:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776756.071129:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776756.071133:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.071141:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.071145:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776781 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.071151:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.071152:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776756.071156:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776756.071160:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776756.071163:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776756.071166:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313143824 00000100:00000001:2.0:1775776756.071169:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313145080 00000100:00000001:2.0:1775776756.071171:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313143824 00000100:00000001:2.0:1775776756.071173:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313145080 00000100:00000001:2.0:1775776756.071175:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776756.071177:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.071179:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776756.071183:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776756/real 1775776756] req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776756.071191:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776756.071193:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776756.071195:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776756.071197:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776756.071199:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776756.071201:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776756.071204:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776756.071212:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.071219:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776756.071220:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.071223:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.071224:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776756.071228:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.071234:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776756.071237:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.071244:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776756.071247:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.071248:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776756.071250:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.071253:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776756.071255:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776756.071259:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776756.071262:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776756.071265:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776756.071269:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776756.071276:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791104:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776756.071280:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.071282:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776756.071285:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc700 x1862033099791104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776781 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776756.071291:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776756.071294:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776756.071296:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775776756.071298:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776756.071300:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776756.071302:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776756.071304:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776756.071306:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:2.0:1775776756.071309:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776756.071311:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776756.071312:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc700. 00000100:00000001:2.0:1775776756.071315:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776756.071316:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.071318:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.071324:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776756.071326:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.071328:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776756.071329:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776756.763894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776756.763913:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776756.763915:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776756.763921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776756.763925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776756.763928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.763930:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776756.763932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776756.763935:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776756.827890:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776756.827895:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776756.827899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776756.827900:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776756.827902:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776756.827902:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776756.827905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776756.827905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776756.827985:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776756.827990:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776756.827995:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776756.891921:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776756.891928:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776756.891930:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776756.891935:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776756.891939:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776756.891943:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776756.891945:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776756.891947:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776756.891949:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776757.339894:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313145080, at 4313145088 00000100:00000040:1.0:1775776757.339897:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313145080, at 4313145088 00000100:00000010:1.0:1775776757.339911:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000010:0.0:1775776757.339914:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:2.0:1775776757.787903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776757.787910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776757.787912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776757.787918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776757.787921:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776757.787925:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776757.787927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776757.787929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776757.787932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776757.851906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776757.851915:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776757.851917:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776757.851920:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776757.851924:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776757.851926:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776757.851928:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776757.851930:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776757.851934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776757.851936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776757.851939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776757.851940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776757.851943:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776757.851945:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776757.852069:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776757.852075:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776757.852080:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776757.915906:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776757.915913:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776757.915915:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776757.915920:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776757.915924:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776757.915928:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776757.915930:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776757.915933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776757.915936:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776758.812323:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776758.812328:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776758.812330:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776758.812334:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776758.812338:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776758.812341:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776758.812344:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776758.812346:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776758.812348:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776758.876361:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776758.876367:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776758.876373:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776758.876375:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776758.876378:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776758.876379:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776758.876382:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776758.876383:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776758.876438:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776758.876442:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776758.876445:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776758.939916:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776758.939924:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776758.939926:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776758.939932:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776758.939936:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776758.939940:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776758.939942:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776758.939945:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776758.939947:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776759.836078:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776759.836086:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776759.836088:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776759.836093:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776759.836109:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776759.836113:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776759.836116:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776759.836118:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776759.836135:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776759.900012:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776759.900017:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776759.901093:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776759.901096:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776759.901102:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776759.901103:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776759.901106:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776759.901107:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776759.902136:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776759.902140:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776759.902144:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776759.964464:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776759.964472:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776759.964474:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776759.964489:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776759.964493:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776759.964496:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776759.964499:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776759.964501:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776759.964503:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776760.859902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776760.859908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776760.859910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776760.859915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776760.859919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776760.859923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776760.859926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776760.859928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776760.859930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923847:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776760.923852:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923857:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776760.923860:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776760.923863:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776760.923867:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923869:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776760.923872:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923874:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923876:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923880:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776760.923881:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776760.923882:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776760.923884:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923885:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776760.923886:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923888:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923889:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923892:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776760.923893:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776760.923894:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776760.923895:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923897:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776760.923898:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923899:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776760.923901:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776760.923995:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776760.923999:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776760.924001:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776760.987889:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776760.987896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776760.987898:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776760.987904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776760.987907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776760.987911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776760.987913:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776760.987916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776760.987918:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776761.180579:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776761.180590:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776761.180593:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776761.180597:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776761.180601:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776761.180609:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18476 00000400:00000200:2.0:1775776761.180619:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776761.180625:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776761.180629:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776761.180632:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.180635:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.180637:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.180639:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776761.180643:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776761.180645:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776761.180648:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776761.180650:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776761.180652:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776761.180655:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776761.180657:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.180659:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.180661:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.180663:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776761.180665:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776761.180668:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776761.180671:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776761.180674:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776761.180677:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776761.180680:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.180683:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776761.180685:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776761.180695:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7df80. 00000020:00000040:2.0:1775776761.180698:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776761.180702:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776761.180704:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.180707:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776761.180709:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776761.180713:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776761.180722:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:2.0:1775776761.180729:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776761.180738:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776761.180749:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776761.180753:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776761.180758:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009deec8f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.180827:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776761.180831:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776761.180833:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776761.180839:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.180847:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776761.180849:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.180852:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776761.180854:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776761.180857:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776761.180860:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099791232:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776761.180864:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776761.180869:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776761.180871:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.180874:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776761.180878:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776761.180880:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776761.180886:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776761.180890:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776761.180896:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776761.180903:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099791232, portal 25 00000100:00000001:3.0:1775776761.180905:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776761.180907:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288718720 : -114841420832896 : ffff978d65e7df80) 00000100:00000040:3.0:1775776761.180911:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.180918:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776761.180920:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776761.180923:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776761.180926:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099791232, offset 0 00000400:00000200:3.0:1775776761.180932:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776761.180939:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.180944:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776761.180949:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776761.180952:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776761.180956:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776761.180959:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776761.180964:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4412:4412:256:4294967295] 192.168.201.139@tcp LPNI seq info [4412:4412:8:4294967295] 00000400:00000200:3.0:1775776761.180968:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.180978:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776761.180983:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.180989:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776761.180995:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776761.180999:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776761.181003:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776761.181007:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776761.181012:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.181015:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776761.181021:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776761.181025:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776761.181030:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776761.181032:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776761.181037:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776761.181041:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776761.181112:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776761.181115:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776761.181118:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776761.181119:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776761.181121:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776761.181124:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776761.181126:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18476 00000400:00000200:2.0:1775776761.181130:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776761.181133:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776761.181135:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776761.181138:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.181140:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.181141:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.181143:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776761.181146:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776761.181147:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776761.181149:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776761.181150:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776761.181152:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776761.181154:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776761.181157:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.181159:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.181161:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.181162:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776761.181164:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776761.181166:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776761.181169:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776761.181170:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776761.181175:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776761.181177:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.181179:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776761.181183:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776761.181185:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.181187:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776761.181191:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c380. 00000020:00000040:2.0:1775776761.181193:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776761.181196:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776761.181198:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.181199:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776761.181201:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776761.181203:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776761.181207:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775776761.181210:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776761.181216:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776761.181223:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776761.181225:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776761.181228:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f231afe] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.181445:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.181450:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.181647:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.181649:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776761.181656:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776761.181660:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776761.181663:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313150198 00000100:00000010:3.0:1775776761.181669:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344300. 00000100:00000001:3.0:1775776761.181675:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776761.181820:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313150198 00000100:00000010:3.0:1775776761.181826:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344a80. 00000100:00000001:3.0:1775776761.181830:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776761.181834:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776761.181837:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.181840:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776761.181845:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776761/real 1775776761] req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776761.181856:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776761.181858:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776761.181860:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776761.181863:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776761.181865:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776761.181868:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776761.181871:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776761.181880:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.181887:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776761.181889:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.181892:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.181894:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776761.181897:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.181904:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776761.181908:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.181915:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776761.181919:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.181920:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.181922:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.181924:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776761.181926:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776761.181929:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776761.181933:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776761.181936:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776761.181941:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776761.181949:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099791232:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776761.181953:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.181955:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.181958:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7df80 x1862033099791232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.181965:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776761.181967:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776761.181969:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776761.181972:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776761.181973:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776761.181975:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776761.181977:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776761.181979:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:3.0:1775776761.181983:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776761.181985:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776761.181987:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7df80. 00000100:00000001:3.0:1775776761.181990:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776761.181991:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.181993:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.182000:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776761.182002:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.182004:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776761.182005:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776761.182007:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776761.182010:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.182016:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776761.182018:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.182020:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776761.182022:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776761.182024:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776761.182027:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791360:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776761.182030:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776761.182033:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776761.182034:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.182036:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776761.182039:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776761.182042:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776761.182046:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776761.182048:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776761.182052:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776761.182056:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099791360, portal 10 00000100:00000001:3.0:1775776761.182057:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776761.182059:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288711552 : -114841420840064 : ffff978d65e7c380) 00000100:00000040:3.0:1775776761.182065:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.182071:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776761.182073:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776761.182075:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776761.182077:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099791360, offset 0 00000400:00000200:3.0:1775776761.182082:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776761.182087:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.182091:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776761.182094:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776761.182097:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776761.182099:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776761.182101:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776761.182104:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4413:4413:256:4294967295] 192.168.201.139@tcp LPNI seq info [4413:4413:8:4294967295] 00000400:00000200:3.0:1775776761.182108:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.182114:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776761.182119:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.182124:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776761.182128:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776761.182131:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776761.182136:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776761.182138:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776761.182141:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.182143:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776761.182146:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776761.182150:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776761.182153:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776761.182155:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776761.182157:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776761.182159:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.182167:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.182170:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.182175:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.182176:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776761.182179:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776761.182183:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776761.182186:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776761.182188:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313148933 00000100:00000001:3.0:1775776761.182190:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313150198 00000100:00000001:3.0:1775776761.182192:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313148933 00000100:00000001:3.0:1775776761.182194:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313150198 00000100:00000001:3.0:1775776761.182196:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776761.182198:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.182199:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776761.182203:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776761/real 1775776761] req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776761.182215:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776761.182217:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776761.182218:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776761.182220:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776761.182222:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776761.182224:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776761.182227:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776761.182234:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.182239:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776761.182241:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.182242:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.182243:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776761.182246:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.182253:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776761.182393:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776761.182397:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776761.182400:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776761.182402:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776761.182404:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776761.182406:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776761.182409:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18476 00000400:00000200:2.0:1775776761.182413:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776761.182417:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776761.182522:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776761.182525:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182528:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.182529:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182531:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776761.182533:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776761.182534:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776761.182536:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776761.182538:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776761.182539:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776761.182541:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776761.182543:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182545:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.182546:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182548:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776761.182550:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776761.182553:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182555:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776761.182557:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776761.182561:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776761.182562:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.182564:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776761.182566:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776761.182568:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.182570:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776761.182574:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f800. 00000020:00000040:2.0:1775776761.182577:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776761.182579:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776761.182581:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.182583:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776761.182584:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776761.182587:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776761.182590:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506000. 00000100:00000001:2.0:1775776761.182594:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776761.182598:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776761.182606:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776761.182608:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776761.182612:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8818441] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776761.182620:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776761.182622:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776761.182625:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776761.182626:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776761.182628:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776761.182630:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776761.182632:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18476 00000400:00000200:2.0:1775776761.182635:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776761.182637:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776761.182639:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776761.182641:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182642:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.182644:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182646:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776761.182648:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776761.182649:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776761.182650:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776761.182652:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776761.182653:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776761.182655:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776761.182658:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182659:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.182661:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182662:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776761.182664:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776761.182666:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182667:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776761.182669:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776761.182671:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776761.182673:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.182674:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776761.182676:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776761.182678:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.182679:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776761.182681:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ed80. 00000020:00000040:2.0:1775776761.182683:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776761.182685:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776761.182686:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.182687:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776761.182688:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776761.182690:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776761.182693:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506c00. 00000100:00000001:2.0:1775776761.182695:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776761.182699:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776761.182705:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776761.182706:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776761.182709:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006375da9b] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776761.182716:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776761.182718:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776761.182721:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776761.182722:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776761.182723:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776761.182725:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776761.182726:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18476 00000400:00000200:2.0:1775776761.182729:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776761.182731:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776761.182732:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776761.182734:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182736:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.182737:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182739:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776761.182741:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776761.182742:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776761.182743:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776761.182744:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776761.182746:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776761.182748:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776761.182750:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182751:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776761.182753:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182754:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776761.182756:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776761.182758:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776761.182760:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776761.182762:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776761.182764:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776761.182765:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.182767:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776761.182769:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776761.182770:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.182772:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776761.182776:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c700. 00000020:00000040:2.0:1775776761.183960:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776761.183966:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776761.183967:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.183970:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776761.183972:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776761.183974:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776761.183981:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:2.0:1775776761.183985:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776761.183992:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776761.184000:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776761.184003:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776761.184006:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbf2ea94] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776761.184014:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776761.184020:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18486) 00000100:00000040:3.0:1775776761.184124:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.184132:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776761.184135:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.184137:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.184139:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.184141:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776761.184144:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776761.184147:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776761.184151:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776761.184154:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776761.184158:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776761.184166:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791360:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776761.184170:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.184172:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.184175:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099791360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.184181:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776761.184183:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776761.184186:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776761.184189:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776761.184191:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776761.184192:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776761.184195:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776761.184197:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:3.0:1775776761.184200:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776761.184202:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776761.184203:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:3.0:1775776761.184206:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776761.184207:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.184209:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.184215:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776761.184217:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.184219:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776761.184221:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776761.184222:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776761.184226:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.184233:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776761.184234:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.184237:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776761.184239:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776761.184241:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776761.184244:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791488:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776761.184247:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776761.184250:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776761.184252:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.184254:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776761.184553:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776761.184555:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776761.184562:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776761.184566:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776761.184572:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776761.184578:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099791488, portal 10 00000100:00000001:3.0:1775776761.184580:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776761.184582:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724992 : -114841420826624 : ffff978d65e7f800) 00000100:00000040:3.0:1775776761.184589:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.184596:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776761.184599:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776761.184601:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776761.184605:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099791488, offset 0 00000400:00000200:3.0:1775776761.184611:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776761.184832:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.184840:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776761.184845:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776761.184848:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776761.184851:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776761.184853:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776761.184857:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4414:4414:256:4294967295] 192.168.201.139@tcp LPNI seq info [4414:4414:8:4294967295] 00000400:00000200:3.0:1775776761.184860:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.184870:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776761.184875:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.184880:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776761.184886:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776761.184889:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776761.184894:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776761.184897:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776761.184900:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.184902:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776761.184906:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776761.184909:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776761.184913:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776761.184915:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776761.184919:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776761.184922:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.184931:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.184933:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.184939:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.184940:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776761.184944:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776761.184948:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776761.184951:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776761.184953:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313148938 00000100:00000001:3.0:1775776761.184956:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313150198 00000100:00000001:3.0:1775776761.184958:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313148938 00000100:00000001:3.0:1775776761.184960:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313150198 00000100:00000001:3.0:1775776761.184963:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776761.184965:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.184967:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776761.184971:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776761/real 1775776761] req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776761.184979:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776761.184981:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776761.184983:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776761.184985:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776761.184986:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776761.184989:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776761.184992:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776761.185000:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185006:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776761.185007:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185009:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.185010:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776761.185013:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185020:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776761.185023:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185029:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776761.185031:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185032:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.185033:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185036:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776761.185037:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776761.185040:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776761.185043:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776761.185045:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776761.185049:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776761.185062:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791488:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776761.185066:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776761.185070:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185075:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776761.185077:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.185079:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776761.185080:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776761.185082:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776761.185084:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791616:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776761.185088:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776761.185090:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776761.185091:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.185093:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776761.185096:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776761.185098:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776761.185101:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776761.185103:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776761.185108:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776761.185111:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099791616, portal 4 00000100:00000001:3.0:1775776761.185113:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776761.185113:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288722304 : -114841420829312 : ffff978d65e7ed80) 00000100:00000040:3.0:1775776761.185119:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185126:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776761.185127:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776761.185130:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776761.185132:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099791616, offset 0 00000400:00000200:3.0:1775776761.185136:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776761.185141:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.185144:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776761.185147:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776761.185149:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776761.185152:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776761.185154:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776761.185156:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4415:4415:256:4294967295] 192.168.201.139@tcp LPNI seq info [4415:4415:8:4294967295] 00000400:00000200:3.0:1775776761.185160:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.185167:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776761.185171:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.185174:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776761.185178:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776761.185181:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776761.185185:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776761.185187:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776761.185191:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.185192:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776761.185195:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776761.185198:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776761.185200:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776761.185202:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776761.185204:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776761.185207:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185214:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.185217:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185223:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.185225:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776761.185227:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776761.185230:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776761.185233:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776761.185236:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313148947 00000100:00000001:3.0:1775776761.185238:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313150198 00000100:00000001:3.0:1775776761.185240:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313148947 00000100:00000001:3.0:1775776761.185242:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313150198 00000100:00000001:3.0:1775776761.185244:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776761.185246:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.185248:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776761.185252:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776761/real 1775776761] req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776761.185274:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776761.185276:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776761.185277:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776761.185279:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776761.185281:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776761.185283:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776761.185287:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776761.185294:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185301:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776761.185302:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185303:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.185305:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776761.185308:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185314:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776761.185317:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185322:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776761.185325:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185326:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.185327:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185329:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776761.185331:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776761.185333:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776761.185336:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776761.185338:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776761.185552:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776761.185562:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791616:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776761.185567:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776761.185571:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185576:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776761.185578:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.185580:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776761.185582:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776761.185584:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776761.185586:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791744:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776761.185590:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776761.185592:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776761.185593:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776761.185595:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776761.185598:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776761.185600:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776761.185604:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776761.185607:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776761.185610:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776761.185613:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099791744, portal 4 00000100:00000001:3.0:1775776761.185616:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776761.185617:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288712448 : -114841420839168 : ffff978d65e7c700) 00000100:00000040:3.0:1775776761.185719:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185726:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776761.185728:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776761.185731:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776761.185733:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099791744, offset 0 00000400:00000200:3.0:1775776761.185737:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776761.185741:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.185745:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776761.185747:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776761.185749:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776761.185751:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776761.185754:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776761.185756:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4416:4416:256:4294967295] 192.168.201.139@tcp LPNI seq info [4416:4416:8:4294967295] 00000400:00000200:3.0:1775776761.185760:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776761.185766:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776761.185770:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.185773:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776761.185777:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776761.185802:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776761.185806:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776761.185808:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776761.185812:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776761.185813:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776761.185816:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776761.185819:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776761.185822:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776761.185824:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776761.185826:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776761.185829:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185842:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.185844:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776786 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185849:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.185851:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776761.185854:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776761.185857:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776761.185860:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776761.185862:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313148939 00000100:00000001:3.0:1775776761.185864:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313150198 00000100:00000001:3.0:1775776761.185866:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313148939 00000100:00000001:3.0:1775776761.185868:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313150198 00000100:00000001:3.0:1775776761.185871:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776761.185873:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.185875:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776761.185878:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776761/real 1775776761] req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776761.185885:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776761.185887:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776761.185888:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776761.185891:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776761.185892:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776761.185895:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776761.185897:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776761.185904:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185911:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776761.185912:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185914:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.185915:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776761.185919:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185925:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776761.185928:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185934:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776761.185936:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185938:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776761.185939:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185940:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776761.185942:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776761.185944:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776761.185947:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776761.185950:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776761.185953:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776761.185959:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099791744:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776761.185963:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185965:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.185968:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f800 x1862033099791488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.185974:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776761.185976:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776761.185978:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776761.185981:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776761.185983:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776761.185984:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776761.185986:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776761.185988:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506000. 02000000:00000001:3.0:1775776761.185991:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776761.185992:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776761.185993:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f800. 00000100:00000001:3.0:1775776761.185996:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776761.185998:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.185999:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.186002:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ed80 x1862033099791616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.186009:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776761.186010:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776761.186011:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776761.186013:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776761.186014:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776761.186015:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776761.186017:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776761.186018:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:3.0:1775776761.186020:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776761.186021:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776761.186022:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ed80. 00000100:00000001:3.0:1775776761.186024:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776761.186025:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.186026:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776761.186029:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c700 x1862033099791744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776786 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776761.186034:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776761.186035:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776761.186036:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776761.186037:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776761.186038:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776761.186040:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776761.186041:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776761.186042:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:3.0:1775776761.186044:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776761.186046:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776761.186047:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c700. 00000100:00000001:3.0:1775776761.186049:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776761.186050:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.186052:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776761.186058:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776761.186059:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776761.186062:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776761.186063:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776761.883899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776761.883908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776761.883910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776761.883915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776761.883919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776761.883923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776761.883926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776761.883929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776761.883931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776761.947906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776761.947913:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776761.947919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776761.947920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776761.947924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776761.947925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776761.947928:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776761.947929:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776761.948022:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776761.948025:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776761.948028:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776762.011898:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776762.011907:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776762.011909:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776762.011913:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776762.011917:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776762.011921:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776762.011924:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776762.011925:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776762.011928:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776762.460101:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313150198, at 4313150208 00000100:00000010:0.0:1775776762.460116:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344300. 00000100:00000040:1.0:1775776762.460210:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313150198, at 4313150208 00000100:00000010:1.0:1775776762.460216:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344a80. 00000100:00000001:2.0:1775776762.908035:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776762.908041:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776762.908043:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776762.908048:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776762.908052:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776762.908055:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776762.908058:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776762.908060:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776762.908063:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776762.971946:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776762.971953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776762.971958:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776762.971959:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776762.971963:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776762.971964:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776762.971968:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776762.971969:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776762.972158:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776762.972162:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776762.972165:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776763.035881:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776763.035888:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776763.035889:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776763.035895:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776763.035898:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776763.035901:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776763.035904:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776763.035906:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776763.035908:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776763.931875:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776763.931880:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776763.931882:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776763.931888:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776763.931891:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776763.931895:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776763.931897:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776763.931900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776763.931903:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776763.995905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776763.995911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776763.995917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776763.995918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776763.995921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776763.995923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776763.995926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776763.995927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776763.996183:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776763.996187:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776763.996190:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776764.059974:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776764.059981:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776764.059983:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776764.059988:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776764.059991:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776764.059995:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776764.059998:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776764.060000:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776764.060002:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776764.955907:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776764.955915:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776764.955917:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776764.955923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776764.955926:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776764.955930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776764.955933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776764.955935:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776764.955937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776765.020288:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776765.020693:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776765.020701:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776765.020702:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776765.020706:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776765.020707:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776765.020711:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776765.020712:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776765.024819:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776765.024825:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776765.024829:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776765.083903:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776765.083911:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776765.083913:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776765.083919:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776765.083923:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776765.083927:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776765.083929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776765.083932:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776765.083934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776765.979977:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776765.979983:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776765.979985:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776765.979990:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776765.979994:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776765.979998:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776765.980000:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776765.980003:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776765.980005:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776766.043917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776766.043922:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776766.043926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776766.043927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776766.043929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776766.043930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776766.043932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776766.043933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776766.043988:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776766.043992:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776766.043995:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776766.107917:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776766.107925:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776766.107927:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776766.107932:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776766.107936:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776766.107940:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.107942:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776766.107945:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.107947:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776766.300757:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776766.300793:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776766.300796:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776766.300800:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776766.300805:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776766.300812:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18481 00000400:00000200:2.0:1775776766.300823:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776766.300829:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776766.300832:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776766.300834:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.300837:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.300839:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.300841:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776766.300845:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776766.300846:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776766.300848:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776766.300851:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776766.300853:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776766.300855:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776766.300857:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.300859:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.300861:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.300862:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776766.300865:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776766.300868:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776766.300870:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776766.300873:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776766.300876:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776766.300879:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.300881:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776766.300883:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776766.300892:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7dc00. 00000020:00000040:2.0:1775776766.300896:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776766.300899:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776766.300901:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.300904:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776766.300906:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776766.300910:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776766.300916:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:2.0:1775776766.300923:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776766.300931:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776766.300941:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776766.300946:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776766.300951:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f1188f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776766.300980:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776766.300983:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776766.300986:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776766.300987:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776766.300989:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776766.300991:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776766.300993:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18481 00000400:00000200:2.0:1775776766.300997:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776766.301001:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776766.301003:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776766.301005:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301007:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.301009:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301011:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776766.301013:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776766.301014:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776766.301015:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776766.301017:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776766.301019:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776766.301021:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776766.301023:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301025:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.301026:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301028:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776766.301029:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776766.301032:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301034:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776766.301036:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776766.301041:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776766.301043:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301044:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776766.301048:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776766.301051:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301052:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776766.301055:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d880. 00000020:00000040:2.0:1775776766.301057:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776766.301059:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776766.301060:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301062:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776766.301063:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776766.301064:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776766.301079:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775776766.301083:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776766.301088:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776766.301095:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776766.301097:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776766.301101:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f88f35f] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776766.301114:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776766.301117:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776766.301120:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776766.301121:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776766.301122:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776766.301124:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776766.301127:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18481 00000400:00000200:2.0:1775776766.301130:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776766.301133:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776766.301135:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776766.301138:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301140:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.301141:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301143:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776766.301145:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776766.301146:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776766.301147:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776766.301148:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776766.301150:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776766.301152:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776766.301154:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301156:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.301157:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301159:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776766.301161:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776766.301163:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301165:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776766.301195:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776766.301198:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776766.301200:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301201:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776766.301203:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776766.301205:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301207:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776766.301211:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f480. 00000020:00000040:2.0:1775776766.301213:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776766.301215:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776766.301217:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301218:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776766.301219:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776766.301221:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776766.301225:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505c00. 00000100:00000001:2.0:1775776766.301227:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776766.301232:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776766.301238:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776766.301240:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776766.301243:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002c91b8b4] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776766.301259:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776766.301262:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776766.301266:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776766.301267:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776766.301269:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776766.301271:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776766.301273:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18481 00000400:00000200:2.0:1775776766.301277:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776766.301279:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776766.301281:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776766.301284:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301286:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.301287:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301289:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776766.301291:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776766.301292:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776766.301293:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776766.301295:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776766.301297:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776766.301298:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776766.301301:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301303:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.301304:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301306:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776766.301308:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776766.301311:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301313:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776766.301315:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776766.301318:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776766.301320:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301321:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776766.301324:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776766.301326:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301327:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776766.301331:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f100. 00000020:00000040:2.0:1775776766.301333:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776766.301335:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776766.301336:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301337:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776766.301339:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776766.301340:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776766.301344:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506800. 00000100:00000001:2.0:1775776766.301347:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776766.301351:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776766.301357:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776766.301359:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776766.301362:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048605922] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776766.301368:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776766.301371:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776766.301374:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776766.301376:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776766.301377:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776766.301379:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776766.301381:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18481 00000400:00000200:2.0:1775776766.301384:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776766.301387:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776766.301389:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776766.301391:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301393:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.301395:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301396:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776766.301398:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776766.301399:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776766.301400:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776766.301402:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776766.301404:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776766.301406:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776766.301408:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301410:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776766.301412:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301413:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776766.301415:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776766.301417:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776766.301420:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776766.301421:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776766.301423:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776766.301425:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301426:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776766.301428:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776766.301430:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301432:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776766.301435:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d500. 00000020:00000040:2.0:1775776766.301438:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776766.301440:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776766.301441:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776766.301442:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776766.301444:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776766.301445:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776766.301448:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506400. 00000100:00000001:2.0:1775776766.301451:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776766.301454:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776766.301461:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776766.301463:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776766.301466:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000855db814] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776766.301472:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776766.301476:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18491) 00000100:00000001:3.0:1775776766.301882:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776766.301887:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776766.301890:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776766.301896:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.301902:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776766.301904:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.301907:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776766.301909:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776766.301911:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776766.301914:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099791872:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776766.301918:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776766.301922:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776766.301925:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.301927:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776766.301932:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776766.301933:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776766.301939:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776766.301942:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776766.301948:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776766.301957:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099791872, portal 25 00000100:00000001:3.0:1775776766.301961:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776766.301963:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288717824 : -114841420833792 : ffff978d65e7dc00) 00000100:00000040:3.0:1775776766.301968:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.301975:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776766.301977:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776766.301980:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776766.301983:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099791872, offset 0 00000400:00000200:3.0:1775776766.301990:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776766.301997:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.302003:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776766.302008:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776766.302012:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776766.302016:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776766.302019:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776766.302023:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4417:4417:256:4294967295] 192.168.201.139@tcp LPNI seq info [4417:4417:8:4294967295] 00000400:00000200:3.0:1775776766.302027:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.302037:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776766.302042:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.302049:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776766.302056:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776766.302059:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776766.302063:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776766.302066:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776766.302070:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.302072:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776766.302078:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776766.302082:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776766.302087:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776766.302090:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776766.302093:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776766.302097:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302106:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.302110:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302115:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.302117:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776766.302122:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776766.302126:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776766.302128:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313155319 00000100:00000010:3.0:1775776766.302133:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344f60. 00000100:00000001:3.0:1775776766.302139:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776766.302157:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313155319 00000100:00000010:3.0:1775776766.302161:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344180. 00000100:00000001:3.0:1775776766.302163:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776766.302182:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776766.302185:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.302187:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776766.302191:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776766/real 1775776766] req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776766.302200:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776766.302201:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776766.302203:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776766.302205:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776766.302207:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776766.302209:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776766.302211:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776766.302218:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302225:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776766.302226:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.302228:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.302230:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776766.302234:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302240:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776766.302244:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302251:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776766.302253:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.302255:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.302256:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.302258:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776766.302261:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776766.302264:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776766.302268:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776766.302271:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776766.302275:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776766.302282:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099791872:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776766.302286:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776766.302289:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302294:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776766.302295:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.302297:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776766.302299:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776766.302301:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776766.302303:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792000:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776766.302306:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776766.302308:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776766.302310:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.302311:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776766.302315:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776766.302317:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776766.302320:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776766.302323:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776766.302326:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776766.302329:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099792000, portal 10 00000100:00000001:3.0:1775776766.302331:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776766.302332:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716928 : -114841420834688 : ffff978d65e7d880) 00000100:00000040:3.0:1775776766.302339:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302344:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776766.302346:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776766.302348:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776766.302351:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099792000, offset 0 00000400:00000200:3.0:1775776766.302354:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776766.302358:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.302361:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776766.302364:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776766.302366:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776766.302368:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776766.302370:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776766.302373:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4418:4418:256:4294967295] 192.168.201.139@tcp LPNI seq info [4418:4418:8:4294967295] 00000400:00000200:3.0:1775776766.302376:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.302381:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776766.302385:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.302389:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776766.302392:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776766.302395:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776766.302399:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776766.302401:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776766.302405:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.302406:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776766.302409:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776766.302412:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776766.302415:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776766.302417:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776766.302419:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776766.302422:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302429:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.302431:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302436:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.302438:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776766.302440:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776766.302443:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776766.302447:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776766.302448:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313154054 00000100:00000001:3.0:1775776766.302451:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313155319 00000100:00000001:3.0:1775776766.302452:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313154054 00000100:00000001:3.0:1775776766.302454:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313155319 00000100:00000001:3.0:1775776766.302456:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776766.302457:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.302459:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776766.302901:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776766/real 1775776766] req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776766.302915:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776766.302917:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776766.302919:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776766.302922:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776766.302923:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776766.302925:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776766.302929:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776766.302937:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302943:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776766.302944:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.302946:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.302948:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776766.302951:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302957:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776766.302961:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.302968:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776766.302971:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.302972:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.302973:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.302975:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776766.302977:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776766.302979:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776766.302984:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776766.302987:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776766.302990:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776766.302998:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792000:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776766.303002:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776766.303005:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.303011:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776766.303012:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.303014:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776766.303016:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776766.303018:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776766.303020:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792128:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776766.303023:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776766.303025:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776766.303027:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.303029:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776766.303033:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776766.303035:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776766.303039:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776766.303042:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776766.303057:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776766.303060:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099792128, portal 10 00000100:00000001:3.0:1775776766.303062:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776766.303063:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724096 : -114841420827520 : ffff978d65e7f480) 00000100:00000040:3.0:1775776766.303070:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.303076:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776766.303078:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776766.303080:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776766.303083:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099792128, offset 0 00000400:00000200:3.0:1775776766.303087:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776766.303092:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.303095:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776766.303098:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776766.303101:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776766.303104:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776766.303106:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776766.303110:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4419:4419:256:4294967295] 192.168.201.139@tcp LPNI seq info [4419:4419:8:4294967295] 00000400:00000200:3.0:1775776766.303114:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.303120:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776766.303124:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.303128:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776766.303132:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776766.303134:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776766.303139:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776766.303141:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776766.303144:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.303146:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776766.303149:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776766.303152:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776766.303155:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776766.303157:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776766.303159:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776766.303162:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.304985:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.304995:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305004:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.305005:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776766.305017:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776766.305022:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776766.305028:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776766.305031:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313154058 00000100:00000001:3.0:1775776766.305034:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313155319 00000100:00000001:3.0:1775776766.305037:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313154058 00000100:00000001:3.0:1775776766.305039:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313155319 00000100:00000001:3.0:1775776766.305042:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776766.305044:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.305047:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776766.305051:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776766/real 1775776766] req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776766.305062:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776766.305064:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776766.305066:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776766.305070:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776766.305072:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776766.305074:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776766.305077:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776766.305086:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305092:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776766.305094:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305096:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.305097:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776766.305100:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305106:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776766.305110:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305117:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776766.305120:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305121:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.305122:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305125:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776766.305127:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776766.305130:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776766.305134:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776766.305137:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776766.305140:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776766.305157:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792128:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776766.305161:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776766.305164:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305184:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776766.305186:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.305189:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776766.305192:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776766.305195:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776766.305198:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792256:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776766.305201:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776766.305204:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776766.305206:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.305208:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776766.305217:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115b000. 02000000:00000001:3.0:1775776766.305219:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776766.305223:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776766.305226:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776766.305230:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776766.305235:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099792256, portal 4 00000100:00000001:3.0:1775776766.305237:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776766.305238:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288723200 : -114841420828416 : ffff978d65e7f100) 00000100:00000040:3.0:1775776766.305244:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305257:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776766.305260:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776766.305262:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776766.305265:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099792256, offset 0 00000400:00000200:3.0:1775776766.305270:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776766.305276:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.305282:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776766.305286:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776766.305290:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776766.305293:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776766.305295:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776766.305299:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4420:4420:256:4294967295] 192.168.201.139@tcp LPNI seq info [4420:4420:8:4294967295] 00000400:00000200:3.0:1775776766.305303:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.305311:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776766.305317:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.305320:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776766.305326:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776766.305329:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776766.305334:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776766.305336:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776766.305353:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.305355:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776766.305359:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776766.305363:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776766.305367:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776766.305368:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776766.305371:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776766.305374:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305382:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.305384:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305397:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.305399:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776766.305401:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776766.305404:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776766.305407:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776766.305408:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313154067 00000100:00000001:3.0:1775776766.305411:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313155319 00000100:00000001:3.0:1775776766.305412:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313154067 00000100:00000001:3.0:1775776766.305414:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313155319 00000100:00000001:3.0:1775776766.305416:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776766.305418:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.305420:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776766.305422:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776766/real 1775776766] req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776766.305430:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776766.305431:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776766.305432:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776766.305434:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776766.305436:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776766.305438:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776766.305440:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776766.305447:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305453:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776766.305454:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305456:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.305457:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776766.305460:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305467:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776766.305470:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305477:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776766.305479:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305480:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.305481:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305483:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776766.305484:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776766.305486:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776766.305489:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776766.305491:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776766.305494:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776766.305501:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792256:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776766.305504:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776766.305507:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305514:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776766.305515:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.305517:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776766.305519:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776766.305521:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776766.305523:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792384:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776766.305526:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776766.305528:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776766.305530:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776766.305531:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776766.305535:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775776766.305537:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776766.305539:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776766.305542:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776766.305544:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776766.305546:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099792384, portal 4 00000100:00000001:3.0:1775776766.305548:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776766.305549:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716032 : -114841420835584 : ffff978d65e7d500) 00000100:00000040:3.0:1775776766.305554:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305559:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776766.305561:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776766.305563:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776766.305565:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099792384, offset 0 00000400:00000200:3.0:1775776766.305569:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776766.305572:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.305575:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776766.305578:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776766.305580:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776766.305582:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776766.305584:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776766.305587:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4421:4421:256:4294967295] 192.168.201.139@tcp LPNI seq info [4421:4421:8:4294967295] 00000400:00000200:3.0:1775776766.305591:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776766.305597:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776766.305601:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.305603:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776766.305606:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776766.305609:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776766.305612:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776766.305615:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776766.305618:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776766.305620:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776766.305622:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776766.305625:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776766.305628:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776766.305629:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776766.305631:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776766.305634:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305641:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.305644:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776791 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305650:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.305651:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776766.305653:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776766.305655:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776766.305658:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776766.305659:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313154058 00000100:00000001:3.0:1775776766.305661:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313155319 00000100:00000001:3.0:1775776766.305663:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313154058 00000100:00000001:3.0:1775776766.305665:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313155319 00000100:00000001:3.0:1775776766.305667:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776766.305669:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.305670:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776766.305674:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776766/real 1775776766] req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776766.305681:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776766.305682:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776766.305683:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776766.305685:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776766.305687:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776766.305689:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776766.305692:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776766.305700:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305706:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776766.305707:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305709:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.305710:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776766.305713:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305719:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776766.305722:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.305729:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776766.305731:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305732:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776766.305733:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305735:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776766.305737:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776766.305739:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776766.305742:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776766.305744:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776766.305747:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776766.305754:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792384:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776766.305757:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.305759:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.306136:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7dc00 x1862033099791872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.306146:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776766.306148:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776766.306151:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776766.306154:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776766.306156:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776766.306158:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776766.306160:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776766.306162:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:3.0:1775776766.306178:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776766.306180:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776766.306181:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7dc00. 00000100:00000001:3.0:1775776766.306185:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776766.306186:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.306188:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.306193:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d880 x1862033099792000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.306200:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776766.306201:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776766.306202:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776766.306204:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776766.306205:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776766.306206:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776766.306208:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776766.306209:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:3.0:1775776766.306211:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776766.306212:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776766.306213:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d880. 00000100:00000001:3.0:1775776766.306214:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776766.306216:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.306217:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.306220:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f480 x1862033099792128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.306226:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776766.306227:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776766.306229:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776766.306230:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776766.306231:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776766.306233:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776766.306234:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776766.306235:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505c00. 02000000:00000001:3.0:1775776766.306237:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776766.306238:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776766.306239:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f480. 00000100:00000001:3.0:1775776766.306241:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776766.306242:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.306243:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.306247:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f100 x1862033099792256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.306252:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776766.306253:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776766.306254:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115b000. 02000000:00000001:3.0:1775776766.306256:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776766.306257:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776766.306258:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776766.306260:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776766.306261:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506800. 02000000:00000001:3.0:1775776766.306263:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776766.306263:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776766.306264:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f100. 00000100:00000001:3.0:1775776766.306266:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776766.306267:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.306268:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776766.306271:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d500 x1862033099792384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776791 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776766.306277:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776766.306278:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776766.306279:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775776766.306280:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776766.306281:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776766.306282:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776766.306284:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776766.306286:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506400. 02000000:00000001:3.0:1775776766.306287:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776766.306288:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776766.306289:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d500. 00000100:00000001:3.0:1775776766.306291:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776766.306292:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.306294:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776766.306301:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776766.306303:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776766.306305:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776766.306307:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776767.003890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776767.003896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776767.003898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776767.003903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776767.003906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776767.003910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776767.003913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776767.003915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776767.003918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776767.068043:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776767.068057:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776767.068071:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776767.068072:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776767.068076:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776767.068077:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776767.068080:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776767.068081:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776767.068204:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776767.068207:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776767.068211:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776767.131905:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776767.131912:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776767.131914:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776767.131920:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776767.131923:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776767.131928:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776767.131931:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776767.131933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776767.131936:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776767.579894:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313155319, at 4313155328 00000100:00000010:0.0:1775776767.579910:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344f60. 00000100:00000040:1.0:1775776767.579922:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313155319, at 4313155328 00000100:00000010:1.0:1775776767.579930:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344180. 00000100:00000001:2.0:1775776768.027869:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776768.027887:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776768.027890:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776768.027896:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776768.027900:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776768.027904:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776768.027906:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776768.027909:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776768.027912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776768.091905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776768.091912:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776768.091915:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776768.091919:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776768.091923:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776768.091925:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776768.091927:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776768.091929:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776768.091934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776768.091935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776768.091939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776768.091940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776768.091944:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776768.091945:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776768.092033:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776768.092037:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776768.092040:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776768.155907:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776768.155915:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776768.155917:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776768.155922:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776768.155926:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776768.155930:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776768.155932:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776768.155935:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776768.155938:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776769.052013:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776769.052021:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776769.052023:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776769.052030:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776769.052034:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776769.052039:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776769.052042:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776769.052045:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776769.052048:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776769.115956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776769.115963:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776769.115971:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776769.115972:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776769.115976:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776769.115977:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776769.115981:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776769.115982:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776769.116052:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776769.116057:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776769.116061:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776769.179971:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776769.179981:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776769.179984:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776769.179991:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776769.179996:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776769.180001:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776769.180004:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776769.180008:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776769.180011:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776770.079307:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776770.079316:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776770.079319:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776770.079326:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776770.079330:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776770.079336:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776770.079339:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776770.079341:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776770.079344:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776770.139913:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776770.139920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776770.139929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776770.139930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776770.139934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776770.139935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776770.139939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776770.139940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776770.140049:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776770.140053:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776770.140056:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776770.203928:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776770.203936:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776770.203939:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776770.203946:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776770.203950:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776770.203954:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776770.203957:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776770.203960:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776770.203963:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.099995:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776771.100005:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776771.100008:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776771.100015:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776771.100020:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776771.100026:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.100030:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776771.100033:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.100036:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164570:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776771.164578:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164585:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776771.164590:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776771.164592:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776771.164596:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164601:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776771.164603:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164605:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164608:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164612:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776771.164614:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776771.164615:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776771.164617:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164619:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776771.164620:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164622:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164623:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164627:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776771.164629:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776771.164630:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776771.164631:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164633:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776771.164634:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164635:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776771.164637:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776771.164745:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776771.164750:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776771.164754:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776771.230545:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776771.230553:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776771.230554:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776771.230560:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776771.230564:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776771.230567:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776771.230569:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776771.230571:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776771.230574:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776771.420205:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776771.420216:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776771.420219:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776771.420223:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776771.420227:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776771.420234:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18486 00000400:00000200:3.0:1775776771.420244:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776771.420250:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776771.420254:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776771.420257:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420260:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420262:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420265:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776771.420269:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776771.420271:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776771.420274:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776771.420277:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776771.420279:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776771.420281:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776771.420284:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420286:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420287:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420290:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776771.420292:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776771.420296:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420298:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776771.420301:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776771.420304:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776771.420307:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420309:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776771.420311:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776771.420320:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776771.420324:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776771.420328:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776771.420329:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420332:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776771.420335:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776771.420338:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776771.420345:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776771.420352:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776771.420361:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776771.420371:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776771.420375:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776771.420380:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776771.420416:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776771.420419:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776771.420421:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776771.420422:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776771.420423:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776771.420425:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776771.420427:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18486 00000400:00000200:3.0:1775776771.420430:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776771.420432:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776771.420433:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776771.420435:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420436:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420438:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420439:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776771.420440:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776771.420441:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776771.420442:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776771.420443:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776771.420444:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776771.420445:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776771.420447:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420448:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420449:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420450:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776771.420452:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776771.420453:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420455:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776771.420456:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776771.420460:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776771.420461:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420462:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776771.420465:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776771.420467:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420468:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776771.420471:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776771.420472:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776771.420473:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776771.420474:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420475:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776771.420476:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776771.420477:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776771.420480:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776771.420482:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776771.420485:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776771.420498:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776771.420499:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776771.420502:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776771.420506:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776771.420508:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776771.420510:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776771.420511:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776771.420512:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776771.420514:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776771.420515:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18486 00000400:00000200:3.0:1775776771.420517:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776771.420518:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776771.420520:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776771.420522:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420523:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420524:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420525:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776771.420527:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776771.420527:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776771.420528:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776771.420529:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776771.420530:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776771.420531:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776771.420533:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420534:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420535:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420536:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776771.420537:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776771.420538:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420539:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776771.420540:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776771.420541:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776771.420542:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420543:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776771.420545:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776771.420546:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420547:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776771.420548:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776771.420550:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776771.420551:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776771.420552:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420552:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776771.420558:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776771.420559:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776771.420562:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776771.420564:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776771.420566:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776771.420571:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776771.420572:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776771.420574:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776771.420578:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776771.420579:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776771.420581:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776771.420581:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776771.420582:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776771.420583:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776771.420585:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18486 00000400:00000200:3.0:1775776771.420587:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776771.420589:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776771.420590:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776771.420592:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420593:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420594:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420595:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776771.420596:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776771.420597:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776771.420598:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776771.420599:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776771.420600:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776771.420601:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776771.420603:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420604:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420605:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420606:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776771.420607:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776771.420609:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420610:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776771.420611:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776771.420612:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776771.420613:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420614:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776771.420615:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776771.420617:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420617:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776771.420619:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776771.420620:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776771.420621:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776771.420622:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420623:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776771.420623:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776771.420624:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776771.420626:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775776771.420628:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776771.420630:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776771.420634:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776771.420635:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776771.420637:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776771.420640:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776771.420642:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776771.420643:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776771.420644:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776771.420645:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776771.420647:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776771.420648:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18486 00000400:00000200:3.0:1775776771.420651:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776771.420654:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776771.420656:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776771.420658:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420660:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420662:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420663:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776771.420665:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776771.420666:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776771.420668:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776771.420670:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776771.420671:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776771.420673:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776771.420676:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420677:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776771.420679:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420681:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776771.420683:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776771.420685:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776771.420688:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776771.420689:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776771.420692:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776771.420694:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420695:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776771.420697:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776771.420699:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420701:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776771.420704:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776771.420707:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776771.420709:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776771.420710:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776771.420711:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776771.420712:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776771.420714:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776771.420717:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776771.420720:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776771.420725:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776771.420731:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776771.420733:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776771.420736:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776771.420744:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776771.420748:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18496) 00000100:00000001:2.0:1775776771.420923:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776771.420928:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776771.420930:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776771.420936:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.420942:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776771.420944:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.420946:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776771.420948:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776771.420950:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776771.420952:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099792512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776771.420954:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776771.420958:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776771.420959:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.420961:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776771.420964:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776771.420965:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776771.420969:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776771.420972:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776771.420976:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776771.420982:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099792512, portal 25 00000100:00000001:2.0:1775776771.420984:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776771.420985:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751253504 : -114841958298112 : ffff978d45dece00) 00000100:00000040:2.0:1775776771.420988:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.420992:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776771.420994:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776771.420996:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776771.420998:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099792512, offset 0 00000400:00000200:2.0:1775776771.421003:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776771.421008:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.421012:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776771.421015:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776771.421018:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776771.421021:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776771.421023:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776771.421026:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4422:4422:256:4294967295] 192.168.201.139@tcp LPNI seq info [4422:4422:8:4294967295] 00000400:00000200:2.0:1775776771.421029:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.421036:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776771.421041:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.421045:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776771.421049:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776771.421052:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776771.421055:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776771.421058:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776771.421060:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.421062:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776771.421066:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776771.421070:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776771.421073:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776771.421074:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776771.421089:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776771.421092:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421098:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.421100:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421105:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.421106:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776771.421110:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776771.421113:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776771.421115:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313160438 00000100:00000010:2.0:1775776771.421118:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bcc0. 00000100:00000001:2.0:1775776771.421123:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776771.421138:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313160438 00000100:00000010:2.0:1775776771.421141:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc60. 00000100:00000001:2.0:1775776771.421144:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776771.421146:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776771.421147:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.421149:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776771.421152:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776771/real 1775776771] req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776771.421162:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776771.421163:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776771.421164:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776771.421166:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776771.421167:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776771.421169:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776771.421170:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776771.421175:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421179:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776771.421180:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421182:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.421183:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776771.421186:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421192:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776771.421197:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421203:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776771.421205:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421206:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.421207:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421209:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776771.421211:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776771.421214:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776771.421218:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776771.421220:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776771.421224:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776771.421233:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099792512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776771.421237:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776771.421240:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421245:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776771.421246:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.421248:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776771.421250:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776771.421252:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776771.421254:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776771.421258:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776771.421259:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776771.421261:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.421263:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776771.421265:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776771.421267:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776771.421271:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776771.421273:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776771.421276:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776771.421279:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099792640, portal 10 00000100:00000001:2.0:1775776771.421281:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776771.421283:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751265152 : -114841958286464 : ffff978d45defb80) 00000100:00000040:2.0:1775776771.421289:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421295:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776771.421297:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776771.421300:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776771.421302:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099792640, offset 0 00000400:00000200:2.0:1775776771.421306:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776771.421310:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.421313:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776771.421316:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776771.421318:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776771.421321:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776771.421323:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776771.421325:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4423:4423:256:4294967295] 192.168.201.139@tcp LPNI seq info [4423:4423:8:4294967295] 00000400:00000200:2.0:1775776771.421330:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.421336:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776771.421340:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.421343:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776771.421347:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776771.421349:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776771.421353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776771.421356:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776771.421360:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.421361:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776771.421364:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776771.421368:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776771.421370:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776771.421372:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776771.421375:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776771.421378:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421384:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.421386:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421390:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.421391:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776771.421393:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776771.421396:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776771.421398:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776771.421399:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313159173 00000100:00000001:2.0:1775776771.421402:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313160438 00000100:00000001:2.0:1775776771.421403:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313159173 00000100:00000001:2.0:1775776771.421405:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313160438 00000100:00000001:2.0:1775776771.421406:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776771.421407:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.421408:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776771.421410:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776771/real 1775776771] req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776771.421414:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776771.421415:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776771.421416:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776771.421418:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776771.421419:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776771.421420:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776771.421422:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776771.421427:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421430:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776771.421431:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421432:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.421432:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776771.421434:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421437:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776771.421439:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421443:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776771.421445:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421446:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.421446:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421448:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776771.421449:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776771.421450:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776771.421452:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776771.421453:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776771.421455:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776771.421459:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776771.421461:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776771.421463:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421466:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776771.421467:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.421468:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776771.421469:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776771.421471:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776771.421472:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776771.421474:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776771.421475:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776771.421476:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.421477:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776771.421479:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776771.421480:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776771.421482:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776771.421483:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776771.421485:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776771.421497:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099792768, portal 10 00000100:00000001:2.0:1775776771.421499:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776771.421500:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257088 : -114841958294528 : ffff978d45dedc00) 00000100:00000040:2.0:1775776771.421503:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421507:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776771.421508:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776771.421510:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776771.421511:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099792768, offset 0 00000400:00000200:2.0:1775776771.421513:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776771.421516:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.421518:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776771.421520:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776771.421521:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776771.421523:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776771.421524:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776771.421526:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4424:4424:256:4294967295] 192.168.201.139@tcp LPNI seq info [4424:4424:8:4294967295] 00000400:00000200:2.0:1775776771.421528:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.421532:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776771.421534:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.421536:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776771.421539:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776771.421540:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776771.421542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776771.421544:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776771.421546:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.421547:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776771.421549:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776771.421551:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776771.421631:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776771.421633:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776771.421635:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776771.421637:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421647:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.421648:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421654:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.421655:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776771.421658:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776771.421661:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776771.421664:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776771.421665:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313159174 00000100:00000001:2.0:1775776771.421668:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313160438 00000100:00000001:2.0:1775776771.421669:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313159174 00000100:00000001:2.0:1775776771.421671:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313160438 00000100:00000001:2.0:1775776771.421673:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776771.421675:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.421676:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776771.421680:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776771/real 1775776771] req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776771.421688:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776771.421689:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776771.421691:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776771.421693:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776771.421694:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776771.421696:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776771.421700:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776771.421708:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421713:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776771.421715:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421717:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.421718:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776771.421720:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421727:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776771.421731:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421738:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776771.421740:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421741:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.421743:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421744:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776771.421746:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776771.421748:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776771.421751:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776771.421753:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776771.421756:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776771.421763:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776771.421767:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776771.421770:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421776:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776771.421805:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.421808:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776771.421810:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776771.421812:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776771.421815:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776771.421818:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776771.421820:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776771.421821:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.421822:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776771.421825:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776771.421826:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776771.421829:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776771.421830:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776771.421833:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776771.421835:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099792896, portal 4 00000100:00000001:2.0:1775776771.421837:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776771.421838:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751263360 : -114841958288256 : ffff978d45def480) 00000100:00000040:2.0:1775776771.421843:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421849:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776771.421851:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776771.421853:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776771.421855:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099792896, offset 0 00000400:00000200:2.0:1775776771.421859:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776771.421863:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.421866:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776771.421869:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776771.421871:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776771.421873:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776771.421875:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776771.421878:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4425:4425:256:4294967295] 192.168.201.139@tcp LPNI seq info [4425:4425:8:4294967295] 00000400:00000200:2.0:1775776771.421882:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.421888:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776771.421893:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.421896:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776771.421899:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776771.421902:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776771.421905:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776771.421908:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776771.421911:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.421913:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776771.421915:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776771.421919:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776771.421921:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776771.421923:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776771.421925:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776771.421929:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421936:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.421939:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421943:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.421944:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776771.421946:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776771.421948:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776771.421950:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776771.421952:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313159184 00000100:00000001:2.0:1775776771.421953:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313160438 00000100:00000001:2.0:1775776771.421954:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313159184 00000100:00000001:2.0:1775776771.421956:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313160438 00000100:00000001:2.0:1775776771.421957:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776771.421958:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.421959:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776771.421961:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776771/real 1775776771] req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776771.421965:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776771.421966:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776771.421968:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776771.421969:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776771.421970:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776771.421971:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776771.421973:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776771.421977:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421980:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776771.421981:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421982:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.421983:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776771.421985:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421988:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776771.421990:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.421994:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776771.421995:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421996:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.421996:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.421997:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776771.421998:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776771.422000:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776771.422001:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776771.422002:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776771.422004:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776771.422011:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099792896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776771.422014:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776771.422015:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422019:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776771.422019:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.422021:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776771.422022:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776771.422023:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776771.422024:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776771.422026:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776771.422027:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776771.422028:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776771.422028:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776771.422032:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776771.422033:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776771.422034:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776771.422036:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776771.422038:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776771.422040:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099793024, portal 4 00000100:00000001:2.0:1775776771.422041:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776771.422042:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751262464 : -114841958289152 : ffff978d45def100) 00000100:00000040:2.0:1775776771.422045:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422048:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776771.422050:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776771.422051:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776771.422052:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099793024, offset 0 00000400:00000200:2.0:1775776771.422054:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776771.422057:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.422059:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776771.422061:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776771.422062:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776771.422064:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776771.422065:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776771.422067:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4426:4426:256:4294967295] 192.168.201.139@tcp LPNI seq info [4426:4426:8:4294967295] 00000400:00000200:2.0:1775776771.422070:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776771.422073:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776771.422075:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.422089:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776771.422093:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776771.422095:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776771.422099:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776771.422101:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776771.422108:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776771.422110:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:2.0:1775776771.422112:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776771.422115:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776771.422118:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:2.0:1775776771.422119:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:2.0:1775776771.422121:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776771.422124:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422131:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.422133:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422139:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.422140:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776771.422143:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776771.422145:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776771.422148:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776771.422150:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313159175 00000100:00000001:2.0:1775776771.422152:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313160438 00000100:00000001:2.0:1775776771.422154:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313159175 00000100:00000001:2.0:1775776771.422156:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313160438 00000100:00000001:2.0:1775776771.422158:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776771.422159:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.422161:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776771.422164:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776771/real 1775776771] req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776771.422171:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776771.422172:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776771.422173:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776771.422175:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776771.422177:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776771.422179:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776771.422181:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776771.422189:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422195:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776771.422196:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422198:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.422199:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776771.422202:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422207:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776771.422210:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422215:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776771.422218:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422219:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776771.422220:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422222:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776771.422223:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776771.422226:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776771.422228:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776771.422230:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776771.422233:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776771.422241:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776771.422244:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422246:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.422249:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dece00 x1862033099792512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422255:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776771.422257:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776771.422260:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776771.422263:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776771.422264:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776771.422265:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776771.422267:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776771.422269:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:2.0:1775776771.422273:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776771.422274:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776771.422276:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dece00. 00000100:00000001:2.0:1775776771.422278:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776771.422279:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422281:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.422284:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45defb80 x1862033099792640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422295:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776771.422296:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776771.422297:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776771.422299:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776771.422300:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776771.422301:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776771.422303:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776771.422304:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:2.0:1775776771.422306:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776771.422308:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776771.422309:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45defb80. 00000100:00000001:2.0:1775776771.422312:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776771.422313:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422314:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.422317:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedc00 x1862033099792768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422323:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776771.422324:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776771.422325:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776771.422327:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776771.422328:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776771.422329:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776771.422331:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776771.422332:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158400. 02000000:00000001:2.0:1775776771.422334:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776771.422335:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776771.422336:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedc00. 00000100:00000001:2.0:1775776771.422338:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776771.422339:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422340:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.422343:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def480 x1862033099792896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422348:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776771.422349:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776771.422351:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776771.422352:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776771.422353:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776771.422354:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776771.422356:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776771.422357:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:2.0:1775776771.422359:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776771.422360:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776771.422361:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def480. 00000100:00000001:2.0:1775776771.422363:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776771.422365:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422366:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776771.422369:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def100 x1862033099793024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776796 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776771.422376:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776771.422376:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776771.422378:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776771.422379:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776771.422380:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776771.422381:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776771.422383:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776771.422384:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158c00. 02000000:00000001:2.0:1775776771.422386:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776771.422387:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776771.422388:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def100. 00000100:00000001:2.0:1775776771.422389:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776771.422391:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422393:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776771.422400:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776771.422402:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776771.422404:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776771.422405:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776772.124008:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776772.124015:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776772.124017:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776772.124021:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776772.124025:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776772.124029:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776772.124032:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776772.124034:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776772.124037:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776772.187861:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776772.187867:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776772.187873:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776772.187874:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776772.187877:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776772.187879:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776772.187882:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776772.187883:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776772.188712:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776772.188715:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776772.188719:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776772.251896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776772.251903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776772.251905:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776772.251910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776772.251913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776772.251917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776772.251920:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776772.251922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776772.251925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776772.699932:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313160438, at 4313160448 00000100:00000010:1.0:1775776772.699947:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000040:0.0:1775776772.700538:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313160438, at 4313160448 00000100:00000010:0.0:1775776772.700545:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:2.0:1775776773.148193:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776773.148199:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776773.148201:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776773.148207:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776773.148210:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776773.148214:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776773.148217:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776773.148220:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776773.148222:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776773.211925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776773.211932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776773.211938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776773.211939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776773.211942:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776773.211943:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776773.211946:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776773.211947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776773.212436:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776773.212440:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776773.212443:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776773.275970:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776773.275977:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776773.275979:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776773.275984:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776773.275987:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776773.275991:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776773.275994:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776773.275996:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776773.275999:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776774.171974:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776774.171981:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776774.171983:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776774.171988:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776774.171991:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776774.171995:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776774.171998:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776774.172000:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776774.172003:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776774.236238:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776774.236245:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776774.236250:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776774.236252:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776774.236255:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776774.236256:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776774.236259:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776774.236260:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776774.236448:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776774.236451:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776774.236454:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776774.299910:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776774.299918:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776774.299920:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776774.299926:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776774.299929:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776774.299933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776774.299935:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776774.299937:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776774.299940:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776775.196067:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776775.196073:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776775.196074:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776775.196079:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776775.196082:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776775.196086:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776775.196088:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776775.196090:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776775.196092:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776775.260303:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776775.260309:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776775.260315:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776775.260316:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776775.260319:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776775.260320:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776775.260323:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776775.260324:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776775.260417:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776775.260420:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776775.260423:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776775.323904:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776775.323911:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776775.323913:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776775.323918:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776775.323921:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776775.323925:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776775.323928:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776775.323930:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776775.323933:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776776.220432:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776776.220440:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776776.220442:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776776.220448:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776776.220451:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776776.220455:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776776.220457:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776776.220459:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776776.220462:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776776.283948:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776776.283955:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776776.283962:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776776.283963:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776776.283967:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776776.283981:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776776.283985:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776776.283986:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776776.284360:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776776.284377:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776776.284380:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776776.347933:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776776.347942:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776776.347945:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776776.347952:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776776.347956:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776776.347961:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776776.347964:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776776.347967:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776776.347971:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776776.540086:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776776.540099:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776776.540102:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776776.540108:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776776.540113:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776776.540123:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18491 00000400:00000200:3.0:1775776776.540137:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776776.540144:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776776.540148:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776776.540153:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540156:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540159:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540162:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776776.540167:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776776.540169:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776776.540172:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776776.540175:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776776.540177:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776776.540180:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776776.540183:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540185:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540187:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540189:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776776.540192:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776776.540196:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540199:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776776.540203:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776776.540207:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776776.540211:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540213:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776776.540216:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776776.540228:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000020:00000040:3.0:1775776776.540233:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776776.540238:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776776.540239:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540242:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776776.540245:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776776.540250:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776776.540260:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776776.540267:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776776.540277:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776776.540289:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776776.540295:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776776.540300:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776776.540334:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776776.540338:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776776.540341:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776776.540342:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776776.540344:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776776.540347:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776776.540349:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18491 00000400:00000200:3.0:1775776776.540353:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776776.540356:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776776.540358:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776776.540361:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540363:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540364:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540366:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776776.540369:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776776.540370:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776776.540372:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776776.540373:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776776.540375:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776776.540377:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776776.540380:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540382:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540383:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540385:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776776.540388:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776776.540391:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540393:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776776.540395:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776776.540401:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776776.540403:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540405:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776776.540410:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776776.540413:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540415:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776776.540419:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775776776.540422:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776776.540424:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776776.540425:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540427:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776776.540429:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776776.540430:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776776.540434:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776776.540437:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776776.540442:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776776.540450:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776776.540452:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776776.540456:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776776.540464:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776776.540466:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776776.540470:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776776.540472:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776776.540473:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776776.540476:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776776.540480:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18491 00000400:00000200:3.0:1775776776.540484:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776776.540487:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776776.540489:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776776.540492:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540494:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540495:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540497:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776776.540500:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776776.540501:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776776.540502:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776776.540504:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776776.540506:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776776.540508:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776776.540510:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540512:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540514:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540516:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776776.540518:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776776.540520:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540522:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776776.540523:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776776.540526:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776776.540528:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540530:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776776.540531:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776776.540534:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540535:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776776.540541:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776776.540543:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776776.540546:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776776.540548:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540550:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776776.540551:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776776.540553:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776776.540557:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776776.540560:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776776.540565:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776776.540574:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776776.540576:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776776.540580:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776776.540587:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776776.540589:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776776.540592:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776776.540593:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776776.540595:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776776.540597:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776776.540599:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18491 00000400:00000200:3.0:1775776776.540603:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776776.540606:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776776.540608:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776776.540610:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540612:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540614:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540616:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776776.540618:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776776.540619:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776776.540620:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776776.540622:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776776.540624:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776776.540627:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776776.540630:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540632:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540633:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540635:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776776.540637:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776776.540640:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540642:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776776.540644:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776776.540647:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776776.540649:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540651:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776776.540653:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776776.540656:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540657:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776776.540661:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776776.540663:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776776.540665:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776776.540667:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540669:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776776.540670:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776776.540672:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776776.540675:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776776.540678:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776776.540753:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776776.540761:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776776.540763:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776776.540766:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776776.540773:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776776.540804:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776776.540808:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776776.540810:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776776.540811:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776776.540814:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776776.540817:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18491 00000400:00000200:3.0:1775776776.540821:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776776.540824:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776776.540826:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776776.540828:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540831:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540833:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540834:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776776.540836:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776776.540838:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776776.540839:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776776.540841:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776776.540843:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776776.540845:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776776.540848:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540849:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776776.540851:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540852:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776776.540854:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776776.540857:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776776.540859:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776776.540861:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776776.540864:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776776.540866:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540868:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776776.540871:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776776.540873:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540875:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776776.540879:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776776.540881:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776776.540884:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776776.540885:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776776.540887:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776776.540888:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776776.540890:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776776.540893:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776776.540897:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776776.540901:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776776.540908:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776776.540910:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776776.540914:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776776.540922:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776776.540928:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18501) 00000100:00000001:2.0:1775776776.541017:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776776.541023:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776776.541026:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776776.541033:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.541041:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776776.541043:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.541047:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776776.541050:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776776.541052:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776776.541056:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099793152:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776776.541060:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776776.541065:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776776.541068:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.541071:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776776.541077:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776776.541079:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776776.541085:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776776.541089:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ce70. 00000400:00000010:2.0:1775776776.541098:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720db28. 00000100:00000200:2.0:1775776776.541106:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099793152, portal 25 00000100:00000001:2.0:1775776776.541109:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776776.541110:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751261568 : -114841958290048 : ffff978d45deed80) 00000100:00000040:2.0:1775776776.541116:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.541124:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776776.541127:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776776.541130:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d330. 00000100:00000200:2.0:1775776776.541134:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099793152, offset 0 00000400:00000200:2.0:1775776776.541141:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776776.541152:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.541158:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776776.541164:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776776.541169:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776776.541173:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776776.541176:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776776.541180:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4427:4427:256:4294967295] 192.168.201.139@tcp LPNI seq info [4427:4427:8:4294967295] 00000400:00000200:2.0:1775776776.541184:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.541196:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776776.541202:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776776.541208:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7900. 00000800:00000200:2.0:1775776776.541216:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776776.541220:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776776.541242:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776776.541246:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776776.541249:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313165558 00000100:00000010:2.0:1775776776.541256:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc00. 00000100:00000001:2.0:1775776776.541264:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776776.541277:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313165558 00000100:00000010:2.0:1775776776.541282:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bcc0. 00000100:00000001:2.0:1775776776.541285:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776776.541288:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776776.541291:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776776.541295:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776776.541299:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.541306:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776776.541307:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.541309:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776776.541311:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776776.541313:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776776.541315:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793280:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776776.541319:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776776.541320:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776776.541322:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.541323:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776776.541327:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776776.541329:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776776.541332:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776776.541335:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601c948. 00000400:00000010:2.0:1775776776.541340:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d110. 00000100:00000200:2.0:1775776776.541343:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099793280, portal 10 00000100:00000001:2.0:1775776776.541345:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776776.541347:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751251712 : -114841958299904 : ffff978d45dec700) 00000100:00000040:2.0:1775776776.541355:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.541361:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776776.541363:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776776.541365:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d220. 00000100:00000200:2.0:1775776776.541367:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099793280, offset 0 00000400:00000200:2.0:1775776776.541372:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776776.541376:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.541380:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776776.541383:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776776.541385:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776776.541387:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776776.541389:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776776.541392:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4428:4428:255:4294967295] 192.168.201.139@tcp LPNI seq info [4428:4428:7:4294967295] 00000400:00000200:2.0:1775776776.541396:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.541401:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776776.541405:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776776.541409:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7500. 00000800:00000200:2.0:1775776776.541413:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776776.541417:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776776.541420:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776776.541424:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776776.541427:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776776.541429:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313164293 00000100:00000001:2.0:1775776776.541432:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313165558 00000100:00000001:2.0:1775776776.541434:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313164293 00000100:00000001:2.0:1775776776.541437:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313165558 00000100:00000001:2.0:1775776776.541439:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776776.541441:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776776.541443:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776776.541446:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.541453:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776776.541454:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.541456:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776776.541457:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776776.541459:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776776.541461:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793408:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776776.541464:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776776.541466:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776776.541467:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.541469:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776776.541473:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776776.541475:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776776.541477:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776776.541480:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601c9a0. 00000400:00000010:2.0:1775776776.541483:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d2a8. 00000100:00000200:2.0:1775776776.541485:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099793408, portal 10 00000100:00000001:2.0:1775776776.541487:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776776.541488:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751262464 : -114841958289152 : ffff978d45def100) 00000100:00000040:2.0:1775776776.541494:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.541501:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776776.541503:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776776.541505:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d550. 00000100:00000200:2.0:1775776776.541507:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099793408, offset 0 00000400:00000200:2.0:1775776776.541512:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776776.541516:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.541519:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776776.541521:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776776.541523:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776776.541525:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776776.541527:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776776.541530:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4429:4429:254:4294967295] 192.168.201.139@tcp LPNI seq info [4429:4429:6:4294967295] 00000400:00000200:2.0:1775776776.541534:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.541539:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776776.541543:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776776.541546:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7100. 00000800:00000200:2.0:1775776776.541550:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776776.541553:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776776.541556:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776776.541558:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776776.541561:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776776.541562:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313164294 00000100:00000001:2.0:1775776776.541564:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313165558 00000100:00000001:2.0:1775776776.541566:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313164294 00000100:00000001:2.0:1775776776.541568:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313165558 00000100:00000001:2.0:1775776776.541570:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776776.541572:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776776.541573:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776776.541577:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.541589:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776776.541590:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.541592:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776776.541593:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776776.541595:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776776.541597:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793536:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776776.541601:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776776.541603:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776776.541604:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.541606:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776776.541609:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776776.541611:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776776.541615:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776776.541618:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601ccb8. 00000400:00000010:2.0:1775776776.541622:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776776.541625:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099793536, portal 4 00000100:00000001:2.0:1775776776.541626:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776776.541627:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751263360 : -114841958288256 : ffff978d45def480) 00000100:00000040:2.0:1775776776.541632:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.541638:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776776.541640:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776776.541642:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720ddd0. 00000100:00000200:2.0:1775776776.541644:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099793536, offset 0 00000400:00000200:2.0:1775776776.541648:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776776.541651:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.541654:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776776.541657:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776776.541659:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776776.541661:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776776.541663:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776776.541666:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4430:4430:253:4294967295] 192.168.201.139@tcp LPNI seq info [4430:4430:5:4294967295] 00000400:00000200:2.0:1775776776.541670:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.541675:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776776.541679:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776776.542826:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7600. 00000800:00000200:2.0:1775776776.542840:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776776.542845:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776776.542851:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776776.542857:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776776.542863:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776776.542866:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313164305 00000100:00000001:2.0:1775776776.542870:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313165558 00000100:00000001:2.0:1775776776.542872:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313164305 00000100:00000001:2.0:1775776776.542874:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313165558 00000100:00000001:2.0:1775776776.542876:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776776.542879:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776776.542883:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776776.542890:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.542897:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776776.542899:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.542907:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776776.542910:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776776.542913:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776776.542916:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793664:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776776.542921:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776776.542925:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776776.542927:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776776.542930:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776776.542937:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776776.542939:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776776.542944:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776776.542948:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776776.542954:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776776.542960:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099793664, portal 4 00000100:00000001:2.0:1775776776.542962:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776776.542963:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257088 : -114841958294528 : ffff978d45dedc00) 00000100:00000040:2.0:1775776776.542970:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776776.542977:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776776.542979:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776776.542981:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720dc38. 00000100:00000200:2.0:1775776776.542986:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099793664, offset 0 00000400:00000200:2.0:1775776776.543002:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776776.543009:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.543016:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776776.543020:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776776.543023:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776776.543027:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776776.543030:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776776.543033:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4431:4431:252:4294967295] 192.168.201.139@tcp LPNI seq info [4431:4431:4:4294967295] 00000400:00000200:2.0:1775776776.543037:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776776.543046:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776776.543051:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776776.543056:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7c00. 00000800:00000200:2.0:1775776776.543060:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776776.543063:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776776.543066:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776776.543069:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776776.543073:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776776.543074:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313164296 00000100:00000001:2.0:1775776776.543077:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313165558 00000100:00000001:2.0:1775776776.543079:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313164296 00000100:00000001:2.0:1775776776.543082:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313165558 00000100:00000001:2.0:1775776776.543084:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776776.543086:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776776.543090:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776776.543094:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776776.543108:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776776.543111:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775776776.543115:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776776.543118:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776776.543121:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776776.543123:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776777.243856:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776777.243865:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776777.243867:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776777.243875:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776777.243880:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776777.243885:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776777.243888:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776777.243891:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776777.243894:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776777.310338:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776777.310344:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776777.310350:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776777.310350:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776777.310354:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776777.310355:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776777.310357:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776777.310358:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776777.311991:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776777.311997:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776777.312002:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776777.371925:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776777.371933:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776777.371936:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776777.371942:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776777.371947:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776777.371953:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776777.371956:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776777.371958:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776777.371962:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776777.820147:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313165558, at 4313165568 00000100:00000010:1.0:1775776777.820166:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000040:0.0:1775776777.820818:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313165558, at 4313165569 00000100:00000010:0.0:1775776777.820826:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000001:2.0:1775776778.270057:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776778.270078:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776778.270080:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776778.270087:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776778.270091:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776778.270097:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776778.270100:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776778.270103:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776778.270106:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776778.331931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776778.331941:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776778.331943:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776778.331947:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776778.331954:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776778.331966:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776778.331969:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776778.331971:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776778.331976:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776778.331977:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776778.331981:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776778.331982:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776778.331986:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776778.331987:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776778.332696:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776778.332700:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776778.332704:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776778.395896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776778.395905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776778.395907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776778.395914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776778.395919:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776778.395924:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776778.395927:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776778.395930:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776778.395934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776779.291954:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776779.291964:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776779.291966:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776779.291974:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776779.291979:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776779.291984:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.291986:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776779.291989:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.291992:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776779.355894:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776779.355899:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776779.355904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776779.355915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776779.355919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776779.355920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776779.355923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776779.355924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776779.356096:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776779.356100:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776779.356103:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776779.419914:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776779.419921:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776779.419922:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776779.419928:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776779.419932:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776779.419944:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776779.419947:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776779.419949:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776779.419952:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1775776779.615128:0:301878:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775776779.615169:0:301878:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775776779.615179:0:301878:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776779.615183:0:301878:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775776779.615187:0:301878:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776779.615191:0:301878:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775776779.615197:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776779.615200:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7900. 00000400:00000200:1.0:1775776779.615206:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776779.615211:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776779.615217:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d330 00000400:00000010:1.0:1775776779.615219:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d330. 00000100:00000001:1.0:1775776779.615223:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776779.615231:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615262:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776779.615267:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615275:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776779.615276:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776779.615285:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776779.615288:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7500. 00000400:00000200:1.0:1775776779.615290:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776779.615293:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776779.615296:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d220 00000400:00000010:1.0:1775776779.615298:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d220. 00000100:00000001:1.0:1775776779.615300:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776779.615304:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615311:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776779.615314:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615320:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776779.615322:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776779.615326:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776779.615328:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7100. 00000400:00000200:1.0:1775776779.615330:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776779.615333:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776779.615336:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d550 00000400:00000010:1.0:1775776779.615337:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d550. 00000100:00000001:1.0:1775776779.615350:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776779.615354:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615361:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776779.615364:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615372:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776779.615373:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776779.615377:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776779.615379:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7600. 00000400:00000200:1.0:1775776779.615381:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776779.615384:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776779.615386:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720ddd0 00000400:00000010:1.0:1775776779.615388:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720ddd0. 00000100:00000001:1.0:1775776779.615390:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776779.615394:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615401:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776779.615404:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615410:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776779.615411:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776779.615415:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776779.615417:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7c00. 00000400:00000200:1.0:1775776779.615420:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776779.615431:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776779.615434:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720dc38 00000400:00000010:1.0:1775776779.615436:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720dc38. 00000100:00000001:1.0:1775776779.615438:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776779.615451:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615459:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776779.615462:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776801 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776779.615468:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776779.615469:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776779.615598:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776779.615602:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776779.615604:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776779.615610:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776776/real 1775776779] req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776779.615622:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776779.615624:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776779.615627:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ce70. 00000400:00000200:2.0:1775776779.615630:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720db28 00000400:00000010:2.0:1775776779.615632:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720db28. 00000100:00000001:2.0:1775776779.615635:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776779.615639:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776779.615648:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615653:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776779.615655:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615658:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.615661:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776779.615664:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615670:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776779.615675:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615682:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776779.615686:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615687:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.615688:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615691:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776779.615693:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776779.615697:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776779.615701:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776779.615704:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776779.615708:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776779.615715:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099793152:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776779.615721:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776779.615724:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776776/real 1775776779] req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776779.615731:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776779.615732:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776779.615734:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601c948. 00000400:00000200:2.0:1775776779.615736:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d110 00000400:00000010:2.0:1775776779.615737:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d110. 00000100:00000001:2.0:1775776779.615739:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776779.615741:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776779.615749:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615756:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776779.615757:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615758:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.615759:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776779.615762:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615769:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776779.615772:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615802:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776779.615804:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615806:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.615807:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615809:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776779.615811:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776779.615813:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776779.615817:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776779.615819:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776779.615822:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776779.615830:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793280:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776779.615834:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776779.615837:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776776/real 1775776779] req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776779.615844:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776779.615846:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776779.615847:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601c9a0. 00000400:00000200:2.0:1775776779.615849:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d2a8 00000400:00000010:2.0:1775776779.615851:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d2a8. 00000100:00000001:2.0:1775776779.615852:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776779.615855:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776779.615863:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615869:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776779.615870:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615871:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.615873:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776779.615876:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615883:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776779.615886:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615907:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776779.615909:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615911:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.615912:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615914:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776779.615915:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776779.615918:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776779.615921:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776779.615923:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776779.615927:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776779.615944:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793408:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776779.615948:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776779.615951:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776776/real 1775776779] req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776779.615959:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776779.615960:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776779.615961:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601ccb8. 00000400:00000200:2.0:1775776779.615964:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776779.615965:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776779.615967:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776779.615970:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776779.615978:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615983:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776779.615985:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.615987:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.615988:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776779.615992:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.615997:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776779.616001:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616007:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776779.616009:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616011:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.616011:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616013:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776779.616014:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776779.616017:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776779.616020:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776779.616022:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776779.616026:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776779.616032:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793536:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776779.616036:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776779.616038:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776776/real 1775776779] req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776779.616045:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776779.616046:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776779.616048:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776779.616050:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776779.616051:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776779.616053:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776779.616056:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776779.616063:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616068:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776779.616069:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616071:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.616072:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776779.616075:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616081:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776779.616084:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616089:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776779.616091:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616092:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776779.616093:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616095:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776779.616096:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776779.616098:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776779.616101:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776779.616103:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776779.616106:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776779.616112:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793664:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776779.616115:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616117:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776779.616120:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deed80 x1862033099793152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616126:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776779.616128:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776779.616132:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776779.616135:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776779.616137:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776779.616139:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776779.616141:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776779.616143:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:2.0:1775776779.616147:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776779.616148:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776779.616150:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deed80. 00000100:00000001:2.0:1775776779.616154:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776779.616155:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616157:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776779.616160:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec700 x1862033099793280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616166:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776779.616167:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776779.616168:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776779.616170:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776779.616171:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776779.616172:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776779.616174:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776779.616175:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:2.0:1775776779.616177:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776779.616178:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776779.616179:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec700. 00000100:00000001:2.0:1775776779.616182:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776779.616183:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616185:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776779.616187:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def100 x1862033099793408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616194:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776779.616195:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776779.616196:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776779.616199:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776779.616200:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776779.616201:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776779.616203:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776779.616204:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:2.0:1775776779.616206:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776779.616207:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776779.616208:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def100. 00000100:00000001:2.0:1775776779.616210:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776779.616211:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616212:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776779.616215:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def480 x1862033099793536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616231:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776779.616232:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776779.616233:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776779.616235:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776779.616237:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776779.616238:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776779.616240:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776779.616241:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:2.0:1775776779.616259:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776779.616260:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776779.616261:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def480. 00000100:00000001:2.0:1775776779.616263:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776779.616265:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616266:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776779.616269:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedc00 x1862033099793664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776801 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776779.616275:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776779.616276:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776779.616277:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776779.616280:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776779.616281:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776779.616282:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776779.616284:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776779.616285:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:2.0:1775776779.616287:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776779.616288:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776779.616290:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedc00. 00000100:00000001:2.0:1775776779.616292:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776779.616293:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616295:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776779.616301:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776779.616303:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776779.616305:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776779.616307:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776780.315909:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776780.315918:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776780.315920:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776780.315956:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776780.315961:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776780.315965:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776780.315967:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776780.315969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776780.315971:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776780.380106:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776780.380113:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776780.380119:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776780.380121:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776780.380125:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776780.380126:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776780.380129:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776780.380130:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776780.380990:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776780.380994:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776780.380998:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776780.444197:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776780.444204:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776780.444206:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776780.444211:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776780.444216:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776780.444220:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776780.444222:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776780.444224:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776780.444227:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.339899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776781.339917:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776781.339919:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776781.339926:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776781.339929:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776781.339933:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.339935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776781.339937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.339941:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403951:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776781.403957:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403963:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776781.403966:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776781.403969:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776781.403973:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403975:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776781.403978:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403979:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403981:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403985:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776781.403986:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776781.403987:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776781.403989:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403990:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776781.403991:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403993:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403995:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.403998:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776781.403999:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776781.404000:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776781.404002:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.404003:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776781.404005:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.404006:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776781.404008:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776781.404452:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776781.404456:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776781.404459:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776781.467925:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776781.467931:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776781.467934:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776781.467939:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776781.467943:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776781.467946:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776781.467948:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776781.467950:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776781.467952:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776781.663829:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776781.663839:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776781.663842:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776781.663846:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776781.663850:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776781.663856:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18496 00000400:00000200:0.0:1775776781.663867:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776781.663881:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776781.663885:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776781.663889:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.663892:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.663894:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.663896:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776781.663909:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776781.663911:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776781.663914:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776781.663916:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776781.663919:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776781.663921:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776781.663923:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.663925:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.663927:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.663928:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776781.663931:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776781.663934:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776781.663937:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776781.663940:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776781.663943:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776781.663946:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.663949:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776781.663952:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776781.663962:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776781.663965:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776781.663969:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776781.663971:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.663973:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776781.663976:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776781.663980:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776781.663989:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776781.663996:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776781.664005:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776781.664015:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776781.664020:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776781.664024:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776781.664056:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776781.664060:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776781.664062:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776781.664064:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776781.664065:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776781.664067:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776781.664070:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18496 00000400:00000200:0.0:1775776781.664073:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776781.664076:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776781.664079:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776781.664081:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664083:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.664085:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664087:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776781.664089:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776781.664090:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776781.664091:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776781.664093:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776781.664095:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776781.664097:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776781.664099:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664101:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.664103:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664105:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776781.664107:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776781.664109:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664112:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776781.664113:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776781.664119:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776781.664121:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664123:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776781.664127:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776781.664130:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664132:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776781.664135:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776781.664137:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776781.664139:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776781.664141:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664142:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776781.664144:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776781.664146:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776781.664149:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776781.664153:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776781.664158:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776781.664167:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776781.664167:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776781.664171:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.664172:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775776781.664176:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664176:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776781.664185:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776781.664191:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776781.664196:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776781.664201:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775776781.664202:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776781.664205:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776781.664205:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776781.664208:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1775776781.664211:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775776781.664212:0:303560:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:2.0:1775776781.664215:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775776781.664217:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18496 02000000:00000001:2.0:1775776781.664220:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776781.664224:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00100000:2.0:1775776781.664228:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099793792:192.168.201.139@tcp:250:kworker.0 00000400:00000200:0.0:1775776781.664229:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776781.664234:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776781.664235:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1775776781.664239:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664241:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775776781.664242:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1775776781.664244:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000001:2.0:1775776781.664245:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664248:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:2.0:1775776781.664250:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1775776781.664251:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776781.664253:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776781.664255:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000010:2.0:1775776781.664255:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 00000020:00000040:0.0:1775776781.664257:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776781.664260:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 02000000:00000001:2.0:1775776781.664260:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776781.664264:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776781.664269:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:0.0:1775776781.664270:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664273:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.664275:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000010:2.0:1775776781.664275:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000100:00000001:0.0:1775776781.664279:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776781.664287:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000010:2.0:1775776781.664287:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00080000:0.0:1775776781.664292:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664294:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776781.664296:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:2.0:1775776781.664298:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099793792, portal 25 02000000:00000001:0.0:1775776781.664299:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776781.664301:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1775776781.664303:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.664304:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741753472 : -114841967798144 : ffff978d454dd880) 00000100:00000001:0.0:1775776781.664306:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776781.664310:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00000040:2.0:1775776781.664313:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775776781.664314:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664318:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776781.664324:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776781.664326:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:0.0:1775776781.664328:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000100:00000040:2.0:1775776781.664332:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 02000000:00000001:0.0:1775776781.664333:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776781.664335:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664337:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1775776781.664337:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 02000000:00000001:0.0:1775776781.664340:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776781.664342:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776781.664346:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776781.664349:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776781.664350:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099793792, offset 0 00000100:00100000:0.0:1775776781.664355:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776781.664362:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664371:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775776781.664372:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776781.664375:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776781.664381:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776781.664382:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776781.664389:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775776781.664395:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776781.664397:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00080000:0.0:1775776781.664399:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776781.664403:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776781.664404:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776781.664405:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776781.664407:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775776781.664408:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1775776781.664412:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18496 00000400:00000200:2.0:1775776781.664415:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776781.664418:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.664422:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4432:4432:256:4294967295] 192.168.201.139@tcp LPNI seq info [4432:4432:8:4294967295] 00000400:00000200:0.0:1775776781.664425:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776781.664429:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775776781.664432:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:0.0:1775776781.664435:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664437:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.664440:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664443:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776781.664447:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:2.0:1775776781.664448:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000020:00000040:0.0:1775776781.664450:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776781.664453:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776781.664456:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000800:00000200:2.0:1775776781.664457:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000020:00000001:0.0:1775776781.664458:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776781.664462:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000800:00000010:2.0:1775776781.664463:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000100:00000040:0.0:1775776781.664465:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664468:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.664470:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664472:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776781.664474:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776781.664477:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776781.664480:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776781.664480:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776781.664485:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1775776781.664486:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 02000000:00000001:0.0:1775776781.664488:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776781.664492:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664496:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:2.1:1775776781.664496:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00100000:0.0:1775776781.664498:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000800:00000200:2.1:1775776781.664501:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000008:00000001:0.0:1775776781.664502:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664506:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775776781.664510:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.664512:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776781.664518:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776781.664522:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776781.664526:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776781.664528:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776781.664532:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776781.664536:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664544:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.664547:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664554:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.664555:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776781.664560:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776781.664563:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776781.664566:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313170681 00000100:00000010:2.0:1775776781.664570:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc60. 00000100:00000001:2.0:1775776781.664575:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776781.664582:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313170681 00000100:00000010:2.0:1775776781.664585:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b1e0. 00000100:00000001:2.0:1775776781.664588:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776781.664590:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776781.664593:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.664595:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776781.664598:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776781/real 1775776781] req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776781.664606:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776781.664608:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776781.664610:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776781.664613:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776781.664614:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776781.664616:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776781.664620:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776781.664627:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664631:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776781.664632:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664634:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.664636:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776781.664638:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664642:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776781.664645:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664649:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776781.664651:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664651:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.664652:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664654:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776781.664655:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776781.664656:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776781.664659:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776781.664660:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776781.664662:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776781.664667:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099793792:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776781.664671:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664672:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.664674:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd880 x1862033099793792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664678:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776781.664680:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776781.664681:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776781.664683:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776781.664684:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776781.664685:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776781.664686:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776781.664688:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f400. 02000000:00000001:2.0:1775776781.664691:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776781.664692:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776781.664694:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd880. 00000100:00000001:2.0:1775776781.664696:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776781.664696:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664698:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664704:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776781.664705:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.664707:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776781.664708:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776781.664709:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776781.664711:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664715:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776781.664716:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.664717:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776781.664719:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776781.664720:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776781.664721:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793920:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776781.664723:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776781.664724:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776781.664725:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.664726:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776781.664728:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776781.664730:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776781.664732:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776781.664734:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776781.664736:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776781.664738:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099793920, portal 10 00000100:00000001:2.0:1775776781.664739:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776781.664740:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741760640 : -114841967790976 : ffff978d454df480) 00000100:00000040:2.0:1775776781.664745:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664748:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776781.664750:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776781.664751:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776781.664752:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099793920, offset 0 00000400:00000200:2.0:1775776781.664754:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776781.664757:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.664759:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776781.664760:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776781.664762:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776781.664763:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776781.664765:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776781.664767:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4433:4433:256:4294967295] 192.168.201.139@tcp LPNI seq info [4433:4433:8:4294967295] 00000400:00000200:2.0:1775776781.664769:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.664773:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776781.664776:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.664808:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776781.664811:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776781.664813:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776781.664816:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776781.664817:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776781.664819:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.664821:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776781.664823:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776781.664825:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776781.664827:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776781.664828:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776781.664829:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776781.664831:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664836:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.664837:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664841:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.664842:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776781.664844:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776781.664847:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776781.664849:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776781.664850:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313169416 00000100:00000001:2.0:1775776781.664852:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313170681 00000100:00000001:2.0:1775776781.664853:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313169416 00000100:00000001:2.0:1775776781.664854:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313170681 00000100:00000001:2.0:1775776781.664855:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776781.664856:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.664857:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776781.664859:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776781/real 1775776781] req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776781.664864:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776781.664865:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776781.664866:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776781.664867:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776781.664868:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776781.664869:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776781.664871:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776781.664876:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664879:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776781.664880:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664881:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.664882:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776781.664886:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664890:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776781.664892:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.664896:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776781.664909:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664911:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.664912:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.664914:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776781.664915:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776781.664918:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776781.664921:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776781.664923:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776781.664927:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776781.664933:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776781.664937:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00100000:2.0:1775776781.664937:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099793920:192.168.201.139@tcp:38:kworker.0 02000000:00000001:0.0:1775776781.664940:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776781.664941:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.664943:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776781.664944:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776781.664946:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776781.664950:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776781.664953:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776781.664959:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776781.664966:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776781.664968:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776781.664971:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776781.664980:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776781.664982:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776781.664985:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776781.664986:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776781.664988:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776781.664990:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776781.664992:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18496 00000400:00000200:0.0:1775776781.664996:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776781.664999:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776781.665010:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776781.665013:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.665015:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.665017:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.665019:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776781.665021:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776781.665022:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776781.665023:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776781.665025:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776781.665027:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776781.665029:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776781.665031:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.665033:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776781.665034:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776781.665036:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776781.665037:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776781.665040:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776781.665042:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776781.665044:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776781.665047:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776781.665049:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.665050:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776781.665052:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776781.665055:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.665056:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776781.665060:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776781.665062:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776781.665064:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776781.665065:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.665068:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776781.665071:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776781.665073:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.665073:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000010:0.0:1775776781.665082:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000040:2.0:1775776781.665084:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776781.665087:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776781.665096:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776781.665112:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776781.665115:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776781.665116:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1775776781.665117:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775776781.665119:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.665121:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775776781.665125:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776781.665125:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776781.665133:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794048:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776781.665141:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776781.665144:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.665144:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776781.665147:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.665150:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1775776781.665151:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18507) 02000000:00000010:2.0:1775776781.665154:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776781.665157:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776781.665161:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776781.665164:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776781.665167:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776781.665171:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099794048, portal 10 00000100:00000001:2.0:1775776781.665173:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776781.665174:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741758848 : -114841967792768 : ffff978d454ded80) 00000100:00000040:2.0:1775776781.665181:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665188:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776781.665190:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776781.665193:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776781.665195:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099794048, offset 0 00000400:00000200:2.0:1775776781.665198:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776781.665202:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.665205:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776781.665208:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776781.665210:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776781.665213:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776781.665215:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776781.665218:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4434:4434:256:4294967295] 192.168.201.139@tcp LPNI seq info [4434:4434:8:4294967295] 00000400:00000200:2.0:1775776781.665222:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.665228:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776781.665232:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.665235:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776781.665239:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776781.665241:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776781.665246:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776781.665248:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776781.665252:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.665254:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776781.665256:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776781.665260:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776781.665262:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776781.665264:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776781.665266:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776781.665269:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665277:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.665280:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665286:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.665288:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776781.665290:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776781.665293:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776781.665296:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776781.665298:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313169418 00000100:00000001:2.0:1775776781.665300:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313170681 00000100:00000001:2.0:1775776781.665302:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313169418 00000100:00000001:2.0:1775776781.665304:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313170681 00000100:00000001:2.0:1775776781.665306:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776781.665308:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.665310:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776781.665313:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776781/real 1775776781] req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776781.665321:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776781.665322:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776781.665324:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776781.665326:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776781.665327:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776781.665329:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776781.665332:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776781.665340:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665346:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776781.665347:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665348:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.665349:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776781.665352:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665359:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776781.665362:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665369:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776781.665371:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665373:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.665374:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665376:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776781.665377:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776781.665379:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776781.665383:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776781.665384:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776781.665388:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776781.665395:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794048:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776781.665399:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665400:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.665403:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df480 x1862033099793920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665408:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776781.665410:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776781.665411:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776781.665414:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776781.665415:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776781.665416:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776781.665418:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776781.665419:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:2.0:1775776781.665422:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776781.665423:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776781.665424:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df480. 00000100:00000001:2.0:1775776781.665427:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776781.665428:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665430:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.665433:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ded80 x1862033099794048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665439:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776781.665440:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776781.665442:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776781.665444:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776781.665445:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776781.665446:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776781.665448:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776781.665450:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f800. 02000000:00000001:2.0:1775776781.665452:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776781.665453:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776781.665455:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776781.665457:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776781.665458:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665459:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665463:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776781.665464:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.665466:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776781.665467:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776781.665469:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776781.665472:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665478:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776781.665479:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.665481:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776781.665482:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776781.665484:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776781.665487:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794176:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776781.665491:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776781.665492:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776781.665494:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.665495:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776781.665498:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776781.665500:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776781.665504:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776781.665506:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776781.665510:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776781.665512:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099794176, portal 4 00000100:00000001:2.0:1775776781.665514:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776781.665515:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741759744 : -114841967791872 : ffff978d454df100) 00000100:00000040:2.0:1775776781.665521:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665532:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776781.665535:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776781.665537:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776781.665539:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099794176, offset 0 00000400:00000200:2.0:1775776781.665543:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776781.665546:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.665549:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776781.665552:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776781.665554:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776781.665556:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776781.665558:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776781.665560:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4435:4435:256:4294967295] 192.168.201.139@tcp LPNI seq info [4435:4435:8:4294967295] 00000400:00000200:2.0:1775776781.665563:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.665569:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776781.665572:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.665575:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776781.665578:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776781.665580:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776781.665584:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776781.665587:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776781.665590:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.665592:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776781.665595:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776781.665598:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776781.665601:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776781.665602:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776781.665605:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776781.665608:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665615:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.665618:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665630:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.665631:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776781.665634:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776781.665636:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776781.665639:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776781.665641:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313169427 00000100:00000001:2.0:1775776781.665643:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313170681 00000100:00000001:2.0:1775776781.665645:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313169427 00000100:00000001:2.0:1775776781.665647:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313170681 00000100:00000001:2.0:1775776781.665649:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776781.665651:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.665653:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776781.665656:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776781/real 1775776781] req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776781.665663:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776781.665664:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776781.665665:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776781.665668:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776781.665669:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776781.665671:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776781.665674:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776781.665682:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665688:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776781.665689:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665691:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.665692:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776781.665695:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665702:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776781.665706:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665712:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776781.665714:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665716:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.665717:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.665718:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776781.665720:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776781.665722:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776781.665725:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776781.665727:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776781.665730:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776781.665737:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794176:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776781.665741:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776781.665744:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.665750:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776781.665751:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.665753:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776781.665754:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776781.665756:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776781.665759:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794304:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776781.665761:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776781.665763:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776781.665765:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776781.665767:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776781.665770:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776781.665771:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776781.665774:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776781.665776:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776781.668088:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776781.668101:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099794304, portal 4 00000100:00000001:2.0:1775776781.668104:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776781.668106:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741761536 : -114841967790080 : ffff978d454df800) 00000100:00000040:2.0:1775776781.668121:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.668130:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776781.668133:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776781.668135:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776781.668139:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099794304, offset 0 00000400:00000200:2.0:1775776781.668145:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776781.668155:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.668162:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776781.668166:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776781.668169:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776781.668172:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776781.668175:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776781.668179:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4436:4436:256:4294967295] 192.168.201.139@tcp LPNI seq info [4436:4436:8:4294967295] 00000400:00000200:2.0:1775776781.668184:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776781.668192:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776781.668198:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.668203:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776781.668210:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776781.668213:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776781.668218:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776781.668222:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776781.668226:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776781.668229:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776781.668235:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776781.668240:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776781.668244:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776781.668246:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776781.668250:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776781.668254:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.668264:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.668268:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.668275:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.668276:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776781.668282:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776781.668286:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776781.668290:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776781.668292:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313169421 00000100:00000001:2.0:1775776781.668296:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313170681 00000100:00000001:2.0:1775776781.668298:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313169421 00000100:00000001:2.0:1775776781.668300:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313170681 00000100:00000001:2.0:1775776781.668302:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776781.668305:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.668308:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776781.668312:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776781/real 1775776781] req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776781.668320:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776781.668321:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776781.668323:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776781.668326:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776781.668327:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776781.668329:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776781.668332:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776781.668339:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.668345:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776781.668347:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.668349:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.668350:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776781.668353:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.668359:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776781.668363:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.668369:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776781.668372:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.668373:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776781.668375:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.668377:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776781.668380:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776781.668383:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776781.668388:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776781.668390:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776781.668393:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776781.668402:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794304:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776781.668406:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.668408:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.668412:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df100 x1862033099794176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.668418:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776781.668421:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776781.668424:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776781.668428:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776781.668430:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776781.668432:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776781.668434:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776781.668437:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:2.0:1775776781.668441:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776781.668442:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776781.668444:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df100. 00000100:00000001:2.0:1775776781.668447:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776781.668448:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.668449:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776781.668453:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df800 x1862033099794304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776781.668459:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776781.668460:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776781.668461:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776781.668463:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776781.668464:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776781.668465:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776781.668467:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776781.668474:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e400. 02000000:00000001:2.0:1775776781.668477:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776781.668478:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776781.668479:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df800. 00000100:00000001:2.0:1775776781.668481:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776781.668482:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.668485:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776781.668491:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776781.668493:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776781.668495:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776781.668496:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776782.363880:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776782.363897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776782.363899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776782.363903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776782.363905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776782.363908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776782.363911:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776782.363912:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776782.363914:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776782.428233:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776782.428239:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776782.428244:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776782.428246:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776782.428249:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776782.428250:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776782.428254:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776782.428255:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776782.428321:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776782.428324:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776782.428328:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776782.491922:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776782.491929:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776782.491931:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776782.491936:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776782.491939:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776782.491943:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776782.491945:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776782.491948:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776782.491950:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776782.939907:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313170681, at 4313170688 00000100:00000040:1.0:1775776782.939912:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313170681, at 4313170688 00000100:00000010:0.0:1775776782.939924:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000010:1.0:1775776782.939926:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:2.0:1775776783.387895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776783.387900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776783.387901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776783.387905:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776783.387907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776783.387910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776783.387912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776783.387914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776783.387916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776783.451899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776783.451905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776783.451911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776783.451913:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776783.451916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776783.451918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776783.451920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776783.451922:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776783.451978:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776783.451983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776783.451985:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776783.515903:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776783.515909:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776783.515911:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776783.515916:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776783.515919:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776783.515923:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776783.515925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776783.515927:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776783.515929:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776784.411956:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776784.411963:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776784.411965:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776784.411970:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776784.411974:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776784.411978:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776784.411980:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776784.411983:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776784.411985:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776784.475908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776784.475914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776784.475919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776784.475920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776784.475923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776784.475924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776784.475927:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776784.475928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776784.476429:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776784.476432:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776784.476435:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776784.539946:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776784.539953:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776784.539955:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776784.539961:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776784.539964:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776784.539968:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776784.539970:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776784.539973:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776784.539975:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776785.435988:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776785.435996:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776785.435998:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776785.436004:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776785.436009:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776785.436014:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776785.436017:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776785.436020:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776785.436023:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776785.499917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776785.499925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776785.499933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776785.499934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776785.499938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776785.499939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776785.499943:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776785.499944:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776785.501907:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776785.501912:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776785.501917:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776785.570051:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776785.570195:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776785.570197:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776785.570203:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776785.570207:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776785.570210:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776785.570213:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776785.570215:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776785.570218:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.459941:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776786.459948:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776786.459950:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776786.459955:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776786.459959:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776786.459962:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.459965:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776786.459967:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.459971:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776786.524737:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776786.524742:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776786.524747:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776786.524748:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776786.524751:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776786.524752:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776786.524756:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776786.524757:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776786.524872:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776786.524875:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776786.524878:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776786.588577:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776786.588585:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776786.588586:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776786.588592:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776786.588595:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776786.588599:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776786.588602:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776786.588604:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776786.588606:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776786.782928:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776786.782986:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776786.782989:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776786.782993:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776786.782997:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776786.783005:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18502 00000400:00000200:3.0:1775776786.783014:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776786.783020:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776786.783023:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776786.783026:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.783030:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.783032:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.783034:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776786.783038:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776786.783041:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776786.783043:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776786.783046:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776786.783048:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776786.783050:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776786.783053:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.783055:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.783057:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.783059:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776786.783062:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776786.783065:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776786.783067:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776786.783070:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776786.783074:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776786.783076:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.783078:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776786.783080:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776786.783089:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776786.783093:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776786.783096:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776786.783098:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.783101:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776786.783103:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776786.783107:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776786.783115:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776786.783121:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776786.783130:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776786.783140:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776786.783145:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776786.783150:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.783298:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776786.783303:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776786.783305:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776786.783311:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.783319:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776786.783321:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.783325:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776786.783327:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776786.783330:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776786.783334:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099794432:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776786.783338:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776786.783342:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776786.783344:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.783347:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776786.783351:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776786.783353:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776786.783358:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776786.783362:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776786.783369:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776786.783377:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099794432, portal 25 00000100:00000001:2.0:1775776786.783379:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776786.783381:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751265152 : -114841958286464 : ffff978d45defb80) 00000100:00000040:2.0:1775776786.783387:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.783400:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776786.783403:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776786.783405:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776786.783409:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099794432, offset 0 00000400:00000200:2.0:1775776786.783415:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776786.783422:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.783427:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776786.783432:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776786.783435:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776786.783439:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776786.783442:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776786.783445:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4437:4437:256:4294967295] 192.168.201.139@tcp LPNI seq info [4437:4437:8:4294967295] 00000400:00000200:2.0:1775776786.783449:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.783459:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776786.783465:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.783470:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776786.783477:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776786.783481:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776786.783486:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776786.783489:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776786.783493:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.783495:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776786.783501:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776786.783505:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776786.783510:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776786.783512:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776786.783516:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776786.783520:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776786.783546:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776786.783551:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776786.783554:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776786.783556:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776786.783557:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776786.786534:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776786.786545:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18502 00000400:00000200:3.0:1775776786.786554:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776786.786569:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776786.786572:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776786.786576:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.786579:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.786581:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.786584:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776786.786587:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776786.786588:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776786.786591:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776786.786594:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776786.786596:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776786.786598:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776786.786600:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.786603:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.786604:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.786606:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776786.786609:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776786.786612:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776786.786615:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776786.786618:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776786.786624:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776786.786627:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.786630:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776786.786642:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776786.786645:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.786647:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776786.786659:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776786.786662:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776786.786666:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776786.786667:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.786670:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776786.786672:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776786.786675:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776786.786682:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775776786.786688:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776786.786695:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776786.786705:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776786.786709:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776786.786714:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.790086:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.790103:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.790120:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.790123:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776786.790137:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776786.790145:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776786.790148:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313175804 00000100:00000010:2.0:1775776786.790156:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bba0. 00000100:00000001:2.0:1775776786.790163:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776786.790175:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313175804 00000100:00000010:2.0:1775776786.790178:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775776786.790181:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776786.790184:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776786.790187:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.790191:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776786.790195:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776786/real 1775776786] req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776786.790208:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776786.790209:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776786.790213:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776786.790216:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776786.790218:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776786.790221:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776786.790225:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776786.790235:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.790243:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776786.790245:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.790247:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.790249:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776786.790253:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.790261:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776786.790266:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.790273:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776786.790277:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.790278:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.790280:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.790282:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776786.790284:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776786.790287:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776786.790291:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776786.790293:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776786.790298:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776786.790307:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099794432:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776786.790311:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.790314:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.790316:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45defb80 x1862033099794432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.790324:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776786.790327:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776786.790331:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776786.790335:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776786.790338:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776786.790340:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776786.790342:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776786.790344:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158800. 02000000:00000001:2.0:1775776786.790348:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776786.790350:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776786.790352:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45defb80. 00000100:00000001:2.0:1775776786.790355:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776786.790356:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.790359:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.790366:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776786.790368:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.790370:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776786.790372:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776786.790374:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776786.790378:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.790385:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776786.790387:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.790390:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776786.790393:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776786.790395:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776786.790398:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794560:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776786.790402:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776786.790406:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776786.790408:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.790410:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776786.790414:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776786.790416:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776786.790421:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776786.790423:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776786.790428:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776786.790433:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099794560, portal 10 00000100:00000001:2.0:1775776786.790435:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776786.790437:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751253504 : -114841958298112 : ffff978d45dece00) 00000100:00000040:2.0:1775776786.790444:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.790451:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776786.790454:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776786.790457:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776786.790460:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099794560, offset 0 00000400:00000200:2.0:1775776786.790466:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776786.790472:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.790478:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776786.790483:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776786.790487:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776786.790490:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776786.790493:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776786.790497:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4438:4438:256:4294967295] 192.168.201.139@tcp LPNI seq info [4438:4438:8:4294967295] 00000400:00000200:2.0:1775776786.790502:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.790510:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776786.790516:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.790521:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776786.790528:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776786.790531:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776786.790537:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776786.790540:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776786.790544:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.790546:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776786.790550:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776786.790555:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776786.790559:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776786.791683:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776786.791690:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776786.791697:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.791707:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.791710:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.791718:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.791720:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776786.791725:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776786.791730:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776786.791733:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776786.791736:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313174542 00000100:00000001:2.0:1775776786.791739:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313175804 00000100:00000001:2.0:1775776786.791741:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313174542 00000100:00000001:2.0:1775776786.791743:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313175804 00000100:00000001:2.0:1775776786.791745:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776786.791748:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.791750:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776786.791753:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776786/real 1775776786] req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776786.791762:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776786.791764:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776786.791766:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776786.791768:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776786.791770:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776786.791772:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776786.791775:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776786.791798:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.791804:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776786.791805:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.792112:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.792115:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776786.792122:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.792129:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776786.792134:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.792142:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776786.792145:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.792146:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.792148:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.792150:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776786.792152:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776786.792155:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776786.792158:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776786.792160:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776786.792164:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776786.792172:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794560:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776786.792176:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.792178:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.792181:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dece00 x1862033099794560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.792187:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776786.792189:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776786.792192:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776786.792195:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776786.792197:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776786.792199:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776786.792200:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776786.792202:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158000. 02000000:00000001:2.0:1775776786.792205:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776786.792206:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776786.792208:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dece00. 00000100:00000001:2.0:1775776786.792210:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776786.792212:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.792214:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.792219:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776786.792220:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.792222:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776786.792224:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.792291:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776786.792296:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776786.792301:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776786.792303:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776786.792305:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776786.792308:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776786.792312:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18502 00000400:00000200:3.0:1775776786.792317:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776786.792321:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776786.792324:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776786.792328:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792330:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.792333:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792335:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776786.792338:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776786.792340:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776786.792343:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776786.792345:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776786.792348:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776786.792351:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776786.792354:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792356:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.792358:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792360:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776786.792363:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776786.792367:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792370:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776786.792372:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776786.792378:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776786.792381:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.792383:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776786.792387:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776786.792390:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.792392:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776786.792398:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776786.792401:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776786.792405:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776786.792407:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.792410:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776786.792412:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776786.792415:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776786.792420:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776786.792426:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776786.792432:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776786.792441:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776786.792446:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776786.792450:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776786.792460:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776786.792463:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776786.792466:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776786.792467:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776786.792469:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776786.792471:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776786.792473:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18502 00000400:00000200:3.0:1775776786.792477:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776786.792480:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776786.792482:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776786.792484:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792487:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.792488:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792490:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776786.792493:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776786.792494:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776786.792495:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776786.792497:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776786.792499:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776786.792501:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776786.792504:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792506:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.792508:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792510:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776786.792512:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776786.792515:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776786.792517:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776786.792518:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776786.792520:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776786.792522:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.792524:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776786.792526:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776786.792529:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.792530:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776786.792534:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776786.792536:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776786.792538:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776786.792539:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.792541:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776786.792542:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776786.792544:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776786.792547:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776786.792550:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776786.792554:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776786.792650:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776786.792653:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776786.792654:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776786.792659:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.792667:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776786.792668:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.792670:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776786.792672:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776786.792674:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776786.792678:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794688:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776786.792681:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776786.792683:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776786.792685:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.792687:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776786.792690:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776786.792692:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776786.792697:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776786.792700:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776786.792704:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776786.792710:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099794688, portal 10 00000100:00000001:2.0:1775776786.792712:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776786.792713:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751259776 : -114841958291840 : ffff978d45dee680) 00000100:00000040:2.0:1775776786.792719:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.792726:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776786.792728:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776786.792730:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776786.792733:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099794688, offset 0 00000400:00000200:2.0:1775776786.792738:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776786.792743:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.792748:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776786.792751:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776786.792754:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776786.792757:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776786.792760:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776786.792764:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4439:4439:256:4294967295] 192.168.201.139@tcp LPNI seq info [4439:4439:8:4294967295] 00000400:00000200:2.0:1775776786.792768:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.792776:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000001:3.0:1775776786.792923:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776786.792926:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776786.792931:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776786.793283:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776786.793286:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776786.793289:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776786.793291:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776786.793293:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776786.793295:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776786.793298:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18502 00000400:00000200:3.0:1775776786.793303:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776786.793306:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776786.793309:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776786.793311:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.793314:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.793316:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.793318:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776786.793320:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776786.793321:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776786.793323:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776786.793325:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776786.793327:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776786.793329:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776786.793331:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.793333:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776786.793335:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776786.793337:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776786.793339:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776786.793341:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776786.793344:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776786.793346:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776786.793349:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776786.793351:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.793353:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776786.793355:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776786.793357:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.793359:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776786.793363:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776786.793365:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776786.793368:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776786.793369:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776786.793371:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776786.793372:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776786.793374:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776786.793379:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775776786.793382:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776786.793387:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776786.793395:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776786.793397:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776786.793401:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776786.793409:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776786.793413:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18512) 00000800:00000200:2.0:1775776786.793465:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.793645:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776786.793653:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776786.793655:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776786.793660:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776786.793663:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776786.793667:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.793669:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776786.793674:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776786.793678:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776786.793683:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776786.793686:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776786.793688:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776786.793692:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.793700:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.793703:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.793709:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.793711:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776786.793715:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776786.793718:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776786.793722:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776786.793725:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313174546 00000100:00000001:2.0:1775776786.793728:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313175804 00000100:00000001:2.0:1775776786.793730:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313174546 00000100:00000001:2.0:1775776786.793732:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313175804 00000100:00000001:2.0:1775776786.793734:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776786.793736:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.793738:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776786.793742:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776786/real 1775776786] req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776786.793751:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776786.793753:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776786.793755:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776786.793757:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776786.793759:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776786.793761:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776786.793765:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776786.793774:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.793799:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776786.793801:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.793803:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.793805:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776786.793992:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794000:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776786.794006:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794013:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776786.794016:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794018:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.794019:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794021:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776786.794023:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776786.794026:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776786.794029:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776786.794032:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776786.794035:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776786.794043:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794688:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776786.794047:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794049:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.794052:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee680 x1862033099794688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794059:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776786.794061:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776786.794063:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776786.794066:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776786.794068:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776786.794069:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776786.794071:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776786.794073:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115ac00. 02000000:00000001:2.0:1775776786.794076:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776786.794078:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776786.794079:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee680. 00000100:00000001:2.0:1775776786.794083:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776786.794084:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794086:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794091:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776786.794092:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.794094:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776786.794096:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776786.794098:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776786.794101:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794108:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776786.794109:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.794112:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776786.794114:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776786.794116:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776786.794119:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794816:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776786.794122:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776786.794124:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776786.794126:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.794128:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776786.794131:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776786.794133:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776786.794138:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776786.794140:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776786.794145:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776786.794148:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099794816, portal 4 00000100:00000001:2.0:1775776786.794150:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776786.794152:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751252608 : -114841958299008 : ffff978d45deca80) 00000100:00000040:2.0:1775776786.794157:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794164:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776786.794167:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776786.794169:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776786.794171:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099794816, offset 0 00000400:00000200:2.0:1775776786.794175:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776786.794179:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.794183:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776786.794186:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776786.794189:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776786.794192:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776786.794194:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776786.794198:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4440:4440:256:4294967295] 192.168.201.139@tcp LPNI seq info [4440:4440:8:4294967295] 00000400:00000200:2.0:1775776786.794202:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.794209:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776786.794214:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.794217:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776786.794221:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776786.794223:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776786.794227:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776786.794229:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776786.794233:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.794234:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776786.794237:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776786.794240:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776786.794243:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776786.794245:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776786.794247:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776786.794251:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794258:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.794261:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794267:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.794268:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776786.794271:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776786.794274:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776786.794278:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776786.794279:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313174556 00000100:00000001:2.0:1775776786.794281:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313175804 00000100:00000001:2.0:1775776786.794283:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313174556 00000100:00000001:2.0:1775776786.794285:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313175804 00000100:00000001:2.0:1775776786.794287:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776786.794289:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.794291:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776786.794295:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776786/real 1775776786] req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776786.794303:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776786.794305:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776786.794306:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776786.794309:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776786.794310:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776786.794312:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776786.794315:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776786.794324:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794330:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776786.794331:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794333:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.794335:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776786.794338:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794345:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776786.794348:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794355:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776786.794358:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794359:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.794360:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794362:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776786.794363:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776786.794366:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776786.794369:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776786.794371:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776786.794374:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776786.794382:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794816:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776786.794385:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776786.794388:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794395:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776786.794396:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.794398:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776786.794400:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776786.794402:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776786.794405:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794944:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776786.794408:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776786.794409:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776786.794411:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776786.794413:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776786.794416:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776786.794418:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776786.794421:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776786.794423:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776786.794426:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776786.794429:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099794944, portal 4 00000100:00000001:2.0:1775776786.794431:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776786.794432:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751250816 : -114841958300800 : ffff978d45dec380) 00000100:00000040:2.0:1775776786.794438:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794451:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776786.794453:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776786.794455:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776786.794457:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099794944, offset 0 00000400:00000200:2.0:1775776786.794461:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776786.794465:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.794468:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776786.794471:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776786.794473:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776786.794476:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776786.794478:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776786.794481:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4441:4441:256:4294967295] 192.168.201.139@tcp LPNI seq info [4441:4441:8:4294967295] 00000400:00000200:2.0:1775776786.794486:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776786.794492:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776786.794496:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.794499:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776786.794503:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776786.794505:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776786.794509:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776786.794512:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776786.794515:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776786.794517:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776786.794520:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776786.794523:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776786.794526:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776786.794527:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776786.794529:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776786.794533:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794540:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.794543:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794555:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.794557:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776786.794559:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776786.794638:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776786.794642:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776786.794644:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313174547 00000100:00000001:2.0:1775776786.794646:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313175804 00000100:00000001:2.0:1775776786.794648:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313174547 00000100:00000001:2.0:1775776786.794649:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313175804 00000100:00000001:2.0:1775776786.794651:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776786.794653:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.794655:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776786.794658:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776786/real 1775776786] req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776786.794666:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776786.794667:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776786.794668:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776786.794671:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776786.794672:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776786.794674:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776786.794677:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776786.794685:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794691:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776786.794692:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794694:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.794695:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776786.794698:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794704:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776786.794707:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794713:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776786.794715:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794717:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776786.794718:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794719:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776786.794721:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776786.794723:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776786.794726:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776786.794728:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776786.794731:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776786.794738:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099794944:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776786.794741:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794743:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.794746:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deca80 x1862033099794816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794752:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776786.794753:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776786.794755:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776786.794758:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776786.794759:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776786.794760:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776786.794762:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776786.794764:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a000. 02000000:00000001:2.0:1775776786.794767:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776786.794768:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776786.794769:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deca80. 00000100:00000001:2.0:1775776786.794771:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776786.794772:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794774:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776786.794776:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec380 x1862033099794944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776786.794806:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776786.794807:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776786.794884:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776786.794888:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776786.794889:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776786.794891:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776786.794892:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776786.794894:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158000. 02000000:00000001:2.0:1775776786.794896:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776786.794897:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776786.794898:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec380. 00000100:00000001:2.0:1775776786.794900:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776786.794901:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794903:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776786.794906:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776786.794907:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776786.794909:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776786.794910:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776787.485019:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776787.485026:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776787.485028:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776787.485034:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776787.485037:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776787.485061:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776787.485064:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776787.485066:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776787.485068:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776787.548148:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776787.548155:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776787.548162:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776787.548164:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776787.548167:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776787.548169:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776787.548171:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776787.548172:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776787.548345:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776787.548348:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776787.548351:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776787.612960:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776787.612967:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776787.612969:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776787.612974:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776787.612978:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776787.612981:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776787.612984:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776787.612986:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776787.612989:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776788.059942:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313175804, at 4313175808 00000100:00000010:0.0:1775776788.059958:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000040:1.0:1775776788.060114:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313175804, at 4313175808 00000100:00000010:1.0:1775776788.060119:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775776788.508132:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776788.508139:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776788.508141:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776788.508146:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776788.508150:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776788.508154:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776788.508164:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776788.508167:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776788.508170:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776788.571995:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776788.572002:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776788.572005:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776788.572008:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776788.572012:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776788.572014:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776788.572016:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776788.572019:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776788.572022:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776788.572024:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776788.572027:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776788.572029:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776788.572032:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776788.572033:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776788.572146:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776788.572149:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776788.572153:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776788.635924:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776788.635931:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776788.635933:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776788.635938:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776788.635942:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776788.635946:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776788.635948:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776788.635956:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776788.635959:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776789.531912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776789.531920:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776789.531922:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776789.531928:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776789.531931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776789.531935:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776789.531948:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776789.531950:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776789.531953:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776789.596181:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776789.596187:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776789.596194:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776789.596195:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776789.596198:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776789.596200:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776789.596203:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776789.596204:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776789.596384:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776789.596387:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776789.596390:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776789.659912:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776789.659918:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776789.659919:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776789.659936:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776789.659939:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776789.659942:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776789.659945:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776789.659947:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776789.659950:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776790.555874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776790.555879:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776790.555880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776790.555884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776790.555888:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776790.555892:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776790.555894:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776790.555897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776790.555899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776790.619915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776790.619924:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776790.619926:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776790.619930:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619932:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776790.619933:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619934:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619936:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776790.619940:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776790.619940:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776790.619941:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619942:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776790.619943:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619944:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619945:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619947:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776790.619948:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776790.619949:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776790.619950:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619951:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776790.619952:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619953:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776790.619954:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776790.620014:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776790.620017:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776790.620021:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776790.684012:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776790.684019:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776790.684022:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776790.684027:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776790.684032:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776790.684036:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776790.684038:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776790.684040:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776790.684043:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.579891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776791.579897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776791.579898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776791.579902:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776791.579904:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776791.579908:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.579909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776791.579911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.579913:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776791.643919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776791.643925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776791.643931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776791.643932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776791.643935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776791.643936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776791.643939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776791.643940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776791.643997:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776791.644002:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776791.644005:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776791.707989:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776791.707997:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776791.707999:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776791.708004:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776791.708008:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776791.708021:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776791.708024:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776791.708027:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776791.708030:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776791.902333:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776791.902342:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776791.902346:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776791.902349:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776791.902353:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776791.902360:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18507 00000400:00000200:0.0:1775776791.902370:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776791.902375:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776791.902377:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776791.902379:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902382:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902384:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902386:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776791.902388:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776791.902390:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776791.902392:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776791.902394:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776791.902396:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776791.902397:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776791.902398:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902399:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902401:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902402:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776791.902405:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776791.902408:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902411:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776791.902415:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776791.902418:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776791.902421:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902424:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776791.902426:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776791.902437:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776791.902441:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776791.902445:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776791.902447:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902450:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776791.902453:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776791.902457:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776791.902465:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776791.902480:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776791.902490:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776791.902501:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776791.902506:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776791.902511:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776791.902543:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776791.902547:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776791.902550:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776791.902551:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776791.902552:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776791.902555:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776791.902558:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18507 00000400:00000200:0.0:1775776791.902562:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776791.902565:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776791.902568:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776791.902570:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902572:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902575:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902577:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776791.902579:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776791.902580:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776791.902582:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776791.902584:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776791.902586:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776791.902588:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776791.902590:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902592:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902594:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902595:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776791.902597:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776791.902600:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902602:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776791.902604:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776791.902610:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776791.902612:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902614:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776791.902618:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776791.902621:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902623:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776791.902626:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776791.902629:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776791.902631:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776791.902632:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902633:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776791.902635:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776791.902637:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776791.902650:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776791.902654:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776791.902659:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776791.902668:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776791.902671:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776791.902675:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776791.902682:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776791.902685:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776791.902688:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776791.902690:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776791.902691:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776791.902694:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776791.902696:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18507 00000400:00000200:0.0:1775776791.902700:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776791.902702:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776791.902704:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776791.902706:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902707:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902708:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902710:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776791.902711:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776791.902712:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776791.902713:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776791.902714:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776791.902715:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776791.902716:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776791.902718:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902731:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902733:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902735:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776791.902737:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776791.902740:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902742:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776791.902744:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776791.902747:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776791.902748:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902751:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776791.902753:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776791.902755:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902757:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776791.902762:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776791.902767:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776791.902769:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776791.902770:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902772:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776791.902773:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776791.902775:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776791.902824:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776791.902827:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776791.902832:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776791.902838:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776791.902841:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776791.902844:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776791.902851:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776791.902853:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776791.902856:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776791.902857:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776791.902858:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776791.902860:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776791.902862:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18507 00000400:00000200:0.0:1775776791.902866:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776791.902868:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776791.902870:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776791.902872:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902874:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902875:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902877:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776791.902879:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776791.902880:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776791.902881:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776791.902883:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776791.902884:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776791.902886:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776791.902888:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902890:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902891:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902893:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776791.902895:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776791.902897:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902899:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776791.902901:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776791.902904:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776791.902906:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902908:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776791.902910:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776791.902912:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902914:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776791.902917:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776791.902919:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776791.902921:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776791.902923:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.902925:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776791.902926:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776791.902928:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776791.902931:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776791.902934:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776791.902937:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776791.902944:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776791.902946:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776791.902949:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776791.902956:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776791.902959:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776791.902962:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776791.902963:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776791.902965:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776791.902966:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776791.902969:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18507 00000400:00000200:0.0:1775776791.902972:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776791.902974:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776791.902976:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776791.902978:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902980:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902981:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902983:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776791.902985:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776791.902986:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776791.902987:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776791.902989:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776791.902991:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776791.902993:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776791.902995:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.902997:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776791.902998:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776791.903000:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776791.903002:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776791.903004:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776791.903006:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776791.903007:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776791.903010:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776791.903011:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.903013:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776791.903015:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776791.903017:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.903018:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776791.903021:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776791.903023:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776791.903025:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776791.903026:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776791.903028:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776791.903029:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776791.903031:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776791.903034:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776791.903037:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776791.903041:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776791.903047:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776791.903049:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776791.903052:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776791.903059:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776791.903066:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18517) 00000100:00000001:2.0:1775776791.903158:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776791.903163:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776791.903165:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776791.903170:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903178:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776791.903180:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.903184:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776791.903187:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776791.903189:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776791.903192:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099795072:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776791.903196:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776791.903200:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776791.903202:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.903205:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776791.903210:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776791.903212:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776791.903217:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776791.903220:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776791.903228:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776791.903235:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099795072, portal 25 00000100:00000001:2.0:1775776791.903237:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776791.903239:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741752576 : -114841967799040 : ffff978d454dd500) 00000100:00000040:2.0:1775776791.903243:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903250:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776791.903253:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776791.903255:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776791.903259:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099795072, offset 0 00000400:00000200:2.0:1775776791.903266:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776791.903273:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.903278:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776791.903283:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776791.903287:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776791.903291:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776791.903294:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776791.903297:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4442:4442:256:4294967295] 192.168.201.139@tcp LPNI seq info [4442:4442:8:4294967295] 00000400:00000200:2.0:1775776791.903302:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.903311:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776791.903317:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.903322:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776791.903328:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776791.903331:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776791.903337:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776791.903340:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776791.903344:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.903346:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776791.903353:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776791.903357:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776791.903363:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776791.903365:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776791.903369:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776791.903373:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903383:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.903385:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903391:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.903393:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776791.903398:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776791.903402:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776791.903404:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313180920 00000100:00000010:2.0:1775776791.903409:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b5a0. 00000100:00000001:2.0:1775776791.903416:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776791.903423:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313180920 00000100:00000010:2.0:1775776791.903425:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b7e0. 00000100:00000001:2.0:1775776791.903428:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776791.903431:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776791.903434:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.903436:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776791.903440:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776791/real 1775776791] req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776791.903448:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776791.903450:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776791.903452:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776791.903454:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776791.903455:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776791.903457:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776791.903460:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776791.903467:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903474:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776791.903476:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.903478:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.903480:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776791.903484:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903490:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776791.903494:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903502:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776791.903504:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.903506:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.903507:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.903509:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776791.903511:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776791.903514:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776791.903517:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776791.903520:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776791.903524:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776791.903530:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099795072:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776791.903535:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776791.903537:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903543:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776791.903544:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.903546:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776791.903548:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776791.903549:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776791.903552:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795200:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776791.903555:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776791.903556:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776791.903558:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.903559:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776791.903563:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776791.903565:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776791.903568:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776791.903570:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776791.903575:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776791.903577:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099795200, portal 10 00000100:00000001:2.0:1775776791.903579:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776791.903580:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757952 : -114841967793664 : ffff978d454dea00) 00000100:00000040:2.0:1775776791.903591:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903598:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776791.903599:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776791.903602:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776791.903604:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099795200, offset 0 00000400:00000200:2.0:1775776791.903608:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776791.903611:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.903614:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776791.903617:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776791.903619:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776791.903621:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776791.903623:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776791.903626:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4443:4443:256:4294967295] 192.168.201.139@tcp LPNI seq info [4443:4443:8:4294967295] 00000400:00000200:2.0:1775776791.903629:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.903635:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776791.903639:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.903642:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776791.903646:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776791.903648:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776791.903652:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776791.903655:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776791.903659:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.903661:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776791.903663:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776791.903667:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776791.903669:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776791.903671:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776791.903673:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776791.903676:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903682:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.903684:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903689:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.903691:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776791.903694:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776791.903697:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776791.903699:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776791.903701:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313179655 00000100:00000001:2.0:1775776791.903703:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313180920 00000100:00000001:2.0:1775776791.903705:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313179655 00000100:00000001:2.0:1775776791.903707:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313180920 00000100:00000001:2.0:1775776791.903709:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776791.903710:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.903712:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776791.903715:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776791/real 1775776791] req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776791.903737:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776791.903738:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776791.903740:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776791.903742:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776791.903743:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776791.903745:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776791.903749:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776791.903755:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903761:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776791.903762:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.903764:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.903765:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776791.903768:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903773:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776791.903776:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903814:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776791.903816:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.903818:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.903819:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.903821:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776791.903824:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776791.903827:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776791.903830:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776791.903833:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776791.903837:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776791.903845:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795200:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776791.903849:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776791.903853:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903859:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776791.903861:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.903863:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776791.903865:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776791.903867:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776791.903870:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795328:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776791.903872:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776791.903874:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776791.903875:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.903877:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776791.903880:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776791.903882:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776791.903885:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776791.903888:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776791.903891:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776791.903893:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099795328, portal 10 00000100:00000001:2.0:1775776791.903896:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776791.903897:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741761536 : -114841967790080 : ffff978d454df800) 00000100:00000040:2.0:1775776791.903902:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903908:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776791.903910:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776791.903913:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776791.903916:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099795328, offset 0 00000400:00000200:2.0:1775776791.903920:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776791.903924:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.903926:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776791.903929:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776791.903931:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776791.903935:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776791.903937:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776791.903940:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4444:4444:256:4294967295] 192.168.201.139@tcp LPNI seq info [4444:4444:8:4294967295] 00000400:00000200:2.0:1775776791.903943:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.903949:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776791.903953:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.903956:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776791.903960:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776791.903962:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776791.903966:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776791.903969:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776791.903972:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.903974:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776791.903976:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776791.903979:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776791.903982:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776791.903984:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776791.903986:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776791.903989:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.903996:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.903999:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904004:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.904006:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776791.904008:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776791.904010:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776791.904013:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776791.904015:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313179657 00000100:00000001:2.0:1775776791.904017:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313180920 00000100:00000001:2.0:1775776791.904019:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313179657 00000100:00000001:2.0:1775776791.904021:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313180920 00000100:00000001:2.0:1775776791.904022:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776791.904024:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.904025:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776791.904028:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776791/real 1775776791] req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776791.904035:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776791.904036:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776791.904037:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776791.904039:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776791.904040:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776791.904042:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776791.904044:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776791.904051:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904057:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776791.904058:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904059:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.904060:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776791.904064:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904070:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776791.904074:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904079:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776791.904081:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904083:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.904084:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904085:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776791.904086:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776791.904089:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776791.904092:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776791.904094:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776791.904097:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776791.904104:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795328:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776791.904107:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776791.904110:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904116:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776791.904117:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.904119:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776791.904121:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776791.904123:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776791.904125:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795456:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776791.904128:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776791.904130:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776791.904131:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.904133:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776791.904137:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776791.904139:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776791.904142:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776791.904145:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776791.904148:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776791.904151:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099795456, portal 4 00000100:00000001:2.0:1775776791.904153:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776791.904154:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741759744 : -114841967791872 : ffff978d454df100) 00000100:00000040:2.0:1775776791.904159:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904165:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776791.904167:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776791.904169:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776791.904171:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099795456, offset 0 00000400:00000200:2.0:1775776791.904175:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776791.904179:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.904181:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776791.904184:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776791.904186:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776791.904188:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776791.904190:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776791.904193:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4445:4445:256:4294967295] 192.168.201.139@tcp LPNI seq info [4445:4445:8:4294967295] 00000400:00000200:2.0:1775776791.904196:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.904202:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776791.904206:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.904209:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776791.904212:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776791.904214:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776791.904218:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776791.904220:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776791.904223:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.904224:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776791.904227:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776791.904230:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776791.904232:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776791.904234:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776791.904236:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776791.904239:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904251:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.904253:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904259:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.904260:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776791.904263:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776791.904266:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776791.904269:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776791.904271:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313179666 00000100:00000001:2.0:1775776791.904273:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313180920 00000100:00000001:2.0:1775776791.904275:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313179666 00000100:00000001:2.0:1775776791.904277:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313180920 00000100:00000001:2.0:1775776791.904279:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776791.904281:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.904283:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776791.904286:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776791/real 1775776791] req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776791.904294:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776791.904295:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776791.904297:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776791.904299:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776791.904300:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776791.904302:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776791.904305:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776791.904311:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904318:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776791.904320:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904322:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.904323:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776791.904326:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904331:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776791.904334:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904339:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776791.904341:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904342:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.904343:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904344:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776791.904346:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776791.904348:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776791.904350:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776791.904353:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776791.904356:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776791.904362:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795456:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776791.904365:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776791.904367:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904373:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776791.904374:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.904375:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776791.904377:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776791.904379:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776791.904381:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795584:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776791.904384:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776791.904386:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776791.904387:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776791.904389:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776791.904392:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776791.904393:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776791.904396:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776791.904398:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776791.904401:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776791.904403:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099795584, portal 4 00000100:00000001:2.0:1775776791.904405:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776791.904406:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741758848 : -114841967792768 : ffff978d454ded80) 00000100:00000040:2.0:1775776791.904411:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904418:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776791.904420:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776791.904422:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776791.904424:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099795584, offset 0 00000400:00000200:2.0:1775776791.904427:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776791.904431:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.904434:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776791.904437:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776791.904439:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776791.904441:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776791.904443:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776791.904446:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4446:4446:256:4294967295] 192.168.201.139@tcp LPNI seq info [4446:4446:8:4294967295] 00000400:00000200:2.0:1775776791.904449:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776791.904454:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776791.904458:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.904461:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776791.904465:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776791.904467:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776791.904470:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776791.904472:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776791.904475:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776791.904477:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776791.904480:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776791.904482:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776791.904485:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776791.904486:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776791.904489:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776791.904492:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904499:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.904501:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904507:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.904508:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776791.904511:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776791.904514:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776791.904516:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776791.904518:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313179657 00000100:00000001:2.0:1775776791.904520:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313180920 00000100:00000001:2.0:1775776791.904522:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313179657 00000100:00000001:2.0:1775776791.904524:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313180920 00000100:00000001:2.0:1775776791.904526:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776791.904528:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.904529:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776791.904532:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776791/real 1775776791] req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776791.904539:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776791.904540:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776791.904542:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776791.904543:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776791.904545:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776791.904547:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776791.904549:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776791.904556:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904562:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776791.904563:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904564:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.904566:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776791.904568:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904574:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776791.904577:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904583:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776791.904585:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904587:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776791.904588:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904590:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776791.904591:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776791.904593:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776791.904597:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776791.904599:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776791.904603:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776791.904614:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795584:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776791.904618:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904620:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.904623:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd500 x1862033099795072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904628:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776791.904631:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776791.904633:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776791.904636:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776791.904638:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776791.904639:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776791.904641:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776791.904643:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775776791.904647:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776791.904648:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776791.904650:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd500. 00000100:00000001:2.0:1775776791.904653:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776791.904654:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904655:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.904658:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dea00 x1862033099795200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904665:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776791.904666:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776791.904667:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776791.904669:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776791.904670:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776791.904671:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776791.904673:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776791.904674:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:2.0:1775776791.904677:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776791.904678:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776791.904680:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dea00. 00000100:00000001:2.0:1775776791.904682:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776791.904683:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904684:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.904688:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df800 x1862033099795328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904693:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776791.904694:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776791.904696:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776791.904698:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776791.904699:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776791.904700:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776791.904701:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776791.904703:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:2.0:1775776791.904705:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776791.904706:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776791.904707:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df800. 00000100:00000001:2.0:1775776791.904709:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776791.904710:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904711:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.904714:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df100 x1862033099795456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904734:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776791.904735:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776791.904737:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776791.904739:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776791.904739:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776791.904740:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776791.904744:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776791.904745:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:2.0:1775776791.904748:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776791.904749:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776791.904750:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df100. 00000100:00000001:2.0:1775776791.904752:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776791.904753:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904754:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776791.904757:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ded80 x1862033099795584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776791.904763:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776791.904764:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776791.904766:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776791.904768:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776791.904769:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776791.904770:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776791.904771:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776791.904772:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:2.0:1775776791.904774:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776791.904775:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776791.904776:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776791.904799:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776791.904801:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904803:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776791.904810:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776791.904812:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776791.904814:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776791.904815:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776792.603863:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776792.603871:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776792.603873:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776792.603880:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776792.603888:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776792.603891:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776792.603894:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776792.603896:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776792.603899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776792.668849:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776792.668855:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776792.668861:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776792.668862:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776792.668866:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776792.668867:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776792.668870:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776792.668871:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776792.669043:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776792.669046:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776792.669049:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776792.731993:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776792.732011:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776792.732013:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776792.732019:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776792.732022:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776792.732026:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776792.732029:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776792.732031:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776792.732034:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776793.179974:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313180920, at 4313180928 00000100:00000010:1.0:1775776793.179988:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000040:0.0:1775776793.181670:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313180920, at 4313180929 00000100:00000010:0.0:1775776793.181681:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000001:2.0:1775776793.628485:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776793.628493:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776793.628495:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776793.628501:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776793.628504:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776793.628509:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776793.628511:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776793.628514:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776793.628517:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776793.693186:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776793.693191:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776793.693196:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776793.693197:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776793.693201:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776793.693202:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776793.693205:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776793.693206:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776793.693265:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776793.693269:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776793.693271:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776793.756812:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776793.756818:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776793.756820:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776793.756826:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776793.756830:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776793.756833:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776793.756836:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776793.756838:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776793.756841:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776794.651912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776794.651919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776794.651921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776794.651926:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776794.651930:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776794.651933:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776794.651935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776794.651938:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776794.651941:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776794.715907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776794.715914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776794.715920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776794.715921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776794.715925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776794.715926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776794.715929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776794.715930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776794.716013:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776794.716017:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776794.716021:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776794.779922:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776794.779929:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776794.779931:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776794.779937:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776794.779940:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776794.779944:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776794.779946:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776794.779948:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776794.779951:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776795.675899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776795.675907:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776795.675913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776795.675919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776795.675923:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776795.675928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776795.675930:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776795.675932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776795.675935:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776795.740198:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776795.740204:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776795.740209:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776795.740211:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776795.740214:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776795.740215:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776795.740218:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776795.740219:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776795.740283:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776795.740287:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776795.740291:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776795.804071:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776795.804080:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776795.804081:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776795.804087:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776795.804091:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776795.804094:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776795.804097:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776795.804099:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776795.804102:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776796.699897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776796.699905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776796.699907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776796.699912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776796.699915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776796.699920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776796.699923:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776796.699936:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776796.699940:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776796.764569:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776796.764575:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776796.764580:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776796.764582:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776796.764586:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776796.764587:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776796.764590:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776796.764591:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776796.765865:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776796.765871:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776796.765876:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776796.827909:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776796.827916:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776796.827917:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776796.827922:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776796.827926:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776796.827929:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776796.827932:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776796.827934:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776796.827937:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776797.021880:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776797.021890:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776797.021893:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776797.021897:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776797.021901:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776797.021909:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18512 00000400:00000200:3.0:1775776797.021918:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776797.021923:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776797.021926:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776797.021929:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.021931:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.021933:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.021935:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776797.021939:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776797.021941:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776797.021944:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776797.021946:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776797.021949:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776797.021951:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776797.021953:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.021956:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.021957:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.021958:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776797.021961:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776797.021964:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776797.021967:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776797.021970:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776797.021973:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776797.021975:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.021978:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776797.021980:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776797.021989:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776797.021993:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776797.021997:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776797.021999:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.022001:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776797.022004:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776797.022008:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776797.022015:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776797.022021:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776797.022029:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776797.022040:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776797.022044:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776797.022049:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.022121:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776797.022125:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776797.022127:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776797.022132:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.022140:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776797.022142:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.022145:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776797.022148:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776797.022150:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776797.022153:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099795712:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776797.022157:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776797.022161:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776797.022163:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.022165:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776797.022169:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776797.022171:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776797.022176:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776797.022179:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776797.022185:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776797.022192:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099795712, portal 25 00000100:00000001:2.0:1775776797.022194:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776797.022196:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751260672 : -114841958290944 : ffff978d45deea00) 00000100:00000040:2.0:1775776797.022200:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.022207:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776797.022210:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776797.022212:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776797.022216:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099795712, offset 0 00000400:00000200:2.0:1775776797.022222:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776797.022230:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.022235:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776797.022239:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776797.022244:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776797.022247:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776797.022250:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776797.022253:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4447:4447:256:4294967295] 192.168.201.139@tcp LPNI seq info [4447:4447:8:4294967295] 00000400:00000200:2.0:1775776797.022257:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.022268:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776797.022274:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.022280:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776797.022286:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776797.022289:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776797.022294:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776797.022297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776797.022301:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.022303:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776797.022309:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776797.022314:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776797.022319:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776797.022321:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776797.022324:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776797.022328:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776797.022351:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776797.022355:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776797.022358:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776797.022359:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776797.022361:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776797.022363:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776797.022366:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18512 00000400:00000200:3.0:1775776797.022369:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776797.022372:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776797.022374:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776797.022376:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.022378:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.022380:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.022382:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776797.022383:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776797.022385:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776797.022386:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776797.022388:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776797.022390:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776797.022391:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776797.022394:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.022396:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.022397:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.022399:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776797.022401:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776797.022403:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776797.022405:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776797.022407:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776797.022412:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776797.022413:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.022415:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776797.022419:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776797.022421:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.022423:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776797.022426:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776797.022428:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776797.022431:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776797.022432:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.022434:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776797.022435:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776797.022437:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776797.022440:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776797.022444:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776797.022448:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776797.022455:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776797.022457:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776797.022460:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.022552:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.022557:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.022565:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.022566:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776797.022574:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776797.022577:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776797.022580:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313186039 00000100:00000010:2.0:1775776797.022585:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b4e0. 00000100:00000001:2.0:1775776797.022591:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776797.022605:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313186039 00000100:00000010:2.0:1775776797.022609:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b3c0. 00000100:00000001:2.0:1775776797.022612:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776797.022615:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776797.022694:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.022696:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776797.022702:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776797/real 1775776797] req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776797.023438:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776797.023440:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776797.023443:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776797.023446:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776797.023448:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776797.023451:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776797.023456:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776797.023465:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.023472:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776797.023473:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.023476:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.023478:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776797.023481:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.023488:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776797.023492:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.023498:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776797.023501:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.023503:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.023504:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.023506:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776797.023508:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776797.023512:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776797.023515:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776797.023517:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776797.023521:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776797.023529:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099795712:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776797.023534:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.023536:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.023539:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099795712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.023545:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776797.023548:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776797.023551:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776797.023554:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776797.023556:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776797.023557:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776797.023560:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776797.023562:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158800. 02000000:00000001:2.0:1775776797.023565:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776797.023567:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776797.023569:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:2.0:1775776797.023572:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776797.023573:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.023575:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.023582:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776797.023584:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.023586:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776797.023588:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776797.023589:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776797.023593:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.023599:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776797.023601:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.023603:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776797.023605:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776797.023607:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776797.023610:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795840:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776797.023614:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776797.023616:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776797.023618:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.023765:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776797.023772:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776797.023775:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776797.023796:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776797.023799:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776797.023805:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776797.023809:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099795840, portal 10 00000100:00000001:2.0:1775776797.023812:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776797.023813:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751254400 : -114841958297216 : ffff978d45ded180) 00000100:00000040:2.0:1775776797.023820:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.023826:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776797.023828:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776797.023831:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776797.023905:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099795840, offset 0 00000400:00000200:2.0:1775776797.023912:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776797.023918:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.023924:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776797.023928:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776797.023931:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776797.023935:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776797.023937:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776797.023941:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4448:4448:256:4294967295] 192.168.201.139@tcp LPNI seq info [4448:4448:8:4294967295] 00000400:00000200:2.0:1775776797.023945:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.023953:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776797.023958:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.023963:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776797.023967:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776797.023971:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776797.023975:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776797.023978:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776797.023982:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.023983:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776797.023987:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776797.023991:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776797.023994:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776797.023996:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776797.023998:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776797.024002:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024009:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.024011:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024016:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.024017:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776797.024021:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776797.024025:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776797.024028:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776797.024030:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313184776 00000100:00000001:2.0:1775776797.024032:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313186039 00000100:00000001:2.0:1775776797.024034:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313184776 00000100:00000001:2.0:1775776797.024036:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313186039 00000100:00000001:2.0:1775776797.024038:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776797.024040:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.024042:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776797.024045:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776797/real 1775776797] req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776797.024053:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776797.024054:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776797.024056:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776797.024058:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776797.024059:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776797.024061:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776797.024064:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776797.024070:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024076:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776797.024077:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.024079:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.024080:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776797.024083:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024088:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776797.024092:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024098:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776797.024100:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.024101:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.024102:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.024104:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776797.024105:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776797.024108:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776797.024112:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776797.024113:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776797.024117:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776797.024123:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795840:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776797.024126:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.024128:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.024130:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099795840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024136:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776797.024137:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776797.024139:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776797.024142:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776797.024143:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776797.024144:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776797.024146:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776797.024148:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:2.0:1775776797.024150:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776797.024151:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776797.024152:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:2.0:1775776797.024155:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776797.024156:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.024158:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.024161:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776797.024163:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.024165:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776797.024166:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.024258:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776797.024260:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.024303:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776797.024308:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776797.024312:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776797.024314:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776797.024316:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776797.024319:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776797.024322:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18512 00000400:00000200:3.0:1775776797.024327:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776797.024331:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776797.024334:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776797.024338:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024341:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.024343:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024345:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776797.024347:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776797.024349:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776797.024351:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776797.024353:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776797.024355:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776797.024357:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776797.024360:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024361:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.024363:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024365:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776797.024367:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776797.024370:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024372:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776797.024374:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776797.024379:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776797.024381:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.024383:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776797.024386:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776797.024389:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.024390:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776797.024395:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776797.024397:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776797.024401:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776797.024402:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.024405:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776797.024406:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776797.024410:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776797.024416:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776797.024421:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776797.024426:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776797.024434:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776797.024437:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776797.024441:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024485:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776797.024487:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776797.024489:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776797.024493:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024500:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776797.024502:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.024504:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776797.024506:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776797.024508:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776797.024510:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795968:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776797.024514:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776797.024516:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776797.024518:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.024520:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776797.024523:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776797.024524:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776797.024528:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776797.024530:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776797.024533:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776797.024536:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099795968, portal 10 00000100:00000001:2.0:1775776797.024538:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776797.024539:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751249920 : -114841958301696 : ffff978d45dec000) 00000100:00000040:2.0:1775776797.024545:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024552:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776797.024554:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776797.024556:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776797.024559:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099795968, offset 0 00000400:00000200:2.0:1775776797.024563:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776797.024567:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.024571:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776797.024573:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776797.024576:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776797.024578:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776797.024580:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776797.024582:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4449:4449:256:4294967295] 192.168.201.139@tcp LPNI seq info [4449:4449:8:4294967295] 00000400:00000200:2.0:1775776797.024586:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.024591:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776797.024595:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.024598:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776797.024602:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776797.024605:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776797.024609:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776797.024612:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776797.024615:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.024616:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776797.024701:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776797.024706:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776797.024710:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776797.024712:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776797.024715:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776797.024719:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776797.024742:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776797.024745:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776797.024748:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776797.024750:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776797.024752:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776797.024754:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776797.024757:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18512 00000400:00000200:3.0:1775776797.024761:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776797.024764:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776797.024767:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776797.024770:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024772:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.024773:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024775:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776797.024796:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776797.024797:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776797.024799:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776797.024801:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776797.024802:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776797.024804:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776797.024807:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024809:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.024811:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024813:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776797.024815:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776797.024818:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776797.024821:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776797.024822:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776797.024825:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776797.024827:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.024829:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776797.024938:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.024943:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.024950:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.024952:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776797.024957:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776797.024960:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776797.024963:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776797.024965:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313184778 00000100:00000001:2.0:1775776797.024967:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313186039 00000100:00000001:2.0:1775776797.024969:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313184778 00000100:00000001:2.0:1775776797.024971:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313186039 00000100:00000001:2.0:1775776797.024973:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776797.024975:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.024977:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776797.024981:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776797/real 1775776797] req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776797.024988:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776797.024989:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776797.024991:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776797.024993:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776797.024995:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776797.024997:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776797.024999:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776797.025007:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.025012:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776797.025013:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.025015:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.025017:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776797.025020:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.025026:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776797.025029:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.025035:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776797.025037:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.025039:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.025040:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.025042:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776797.025043:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776797.025045:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776797.025049:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776797.025051:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776797.025054:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776797.025061:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099795968:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776797.025065:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.025067:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.025070:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099795968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.025075:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776797.025077:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776797.025079:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776797.025081:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776797.025082:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776797.025084:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776797.025085:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776797.025087:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:2.0:1775776797.025090:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776797.025091:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776797.025093:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:2.0:1775776797.025095:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776797.025096:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.025098:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.025102:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776797.025104:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.025106:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776797.025107:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776797.025145:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776797.025149:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.025151:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776797.025155:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776797.025157:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776797.025160:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776797.025161:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.025163:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776797.025165:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776797.025167:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776797.025353:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776797.025359:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776797.025365:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776797.025372:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776797.025375:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776797.025378:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776797.025396:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776797.025399:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776797.025402:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776797.025403:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776797.025405:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776797.025407:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776797.025410:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18512 00000400:00000200:3.0:1775776797.025413:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776797.025417:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776797.025419:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776797.025421:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.025424:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.025425:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.025427:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776797.025429:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776797.025430:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776797.025432:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776797.025433:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776797.025435:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776797.025437:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776797.025439:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.025441:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776797.025442:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776797.025444:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776797.025446:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776797.025448:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776797.025451:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776797.025452:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776797.025456:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776797.025458:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.025460:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776797.025462:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776797.025463:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.025465:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776797.025469:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776797.025471:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776797.025473:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776797.025474:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776797.025475:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776797.025477:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776797.025478:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776797.025482:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776797.025484:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776797.025488:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776797.025495:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776797.025497:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776797.025500:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776797.025506:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776797.025511:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18522) 00000100:00000001:2.0:1775776797.025566:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776797.025568:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776797.025569:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776797.025574:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.025580:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776797.025581:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.025583:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776797.025585:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776797.025587:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776797.025590:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796096:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776797.025593:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776797.025595:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776797.025597:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.025599:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776797.025603:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776797.025605:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776797.025609:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776797.025612:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776797.025920:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776797.025928:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099796096, portal 4 00000100:00000001:2.0:1775776797.025930:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776797.025931:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751255296 : -114841958296320 : ffff978d45ded500) 00000100:00000040:2.0:1775776797.025938:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.025944:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776797.025947:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776797.025950:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776797.025953:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099796096, offset 0 00000400:00000200:2.0:1775776797.025957:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776797.025962:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.025966:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776797.025970:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776797.025973:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776797.025976:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776797.025978:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776797.025982:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4450:4450:256:4294967295] 192.168.201.139@tcp LPNI seq info [4450:4450:8:4294967295] 00000400:00000200:2.0:1775776797.025986:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.025993:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776797.025997:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.026001:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776797.026006:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776797.026009:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776797.026013:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776797.026016:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776797.026019:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.026021:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776797.026025:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776797.026029:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776797.026031:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776797.026033:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776797.026035:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776797.026038:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026045:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.026048:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026053:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.026055:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776797.026058:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776797.026060:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776797.026063:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776797.026065:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313184788 00000100:00000001:2.0:1775776797.026067:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313186039 00000100:00000001:2.0:1775776797.026069:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313184788 00000100:00000001:2.0:1775776797.026070:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313186039 00000100:00000001:2.0:1775776797.026072:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776797.026074:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.026076:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776797.026080:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776797/real 1775776797] req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776797.026087:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776797.026088:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776797.026090:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776797.026092:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776797.026093:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776797.026096:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776797.026098:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776797.026105:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026111:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776797.026112:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026114:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.026116:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776797.026119:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026125:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776797.026128:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026134:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776797.026137:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026138:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.026139:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026141:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776797.026143:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776797.026145:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776797.026149:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776797.026151:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776797.026154:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776797.026161:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796096:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776797.026164:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776797.026168:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026173:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776797.026175:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.026177:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776797.026178:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776797.026180:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776797.026183:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796224:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776797.026186:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776797.026188:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776797.026189:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776797.026191:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776797.026194:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776797.026196:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776797.026199:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776797.026201:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776797.026204:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776797.026206:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099796224, portal 4 00000100:00000001:2.0:1775776797.026208:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776797.026209:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751258880 : -114841958292736 : ffff978d45dee300) 00000100:00000040:2.0:1775776797.026214:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026219:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776797.026221:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776797.026223:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776797.026226:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099796224, offset 0 00000400:00000200:2.0:1775776797.026229:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776797.026232:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.026235:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776797.026238:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776797.026240:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776797.026243:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776797.026245:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776797.026248:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4451:4451:256:4294967295] 192.168.201.139@tcp LPNI seq info [4451:4451:8:4294967295] 00000400:00000200:2.0:1775776797.026252:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776797.026259:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776797.026263:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.026266:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776797.026270:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776797.026272:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776797.026276:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776797.026278:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776797.026282:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776797.026284:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776797.026287:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776797.026290:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776797.026292:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776797.026294:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776797.026296:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776797.026299:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026306:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.026378:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776822 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026385:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.026387:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776797.026391:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776797.026394:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776797.026397:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776797.026399:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313184779 00000100:00000001:2.0:1775776797.026401:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313186039 00000100:00000001:2.0:1775776797.026403:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313184779 00000100:00000001:2.0:1775776797.026405:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313186039 00000100:00000001:2.0:1775776797.026407:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776797.026409:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.026411:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776797.026414:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776797/real 1775776797] req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776797.026421:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776797.026422:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776797.026424:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776797.026426:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776797.026427:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776797.026429:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776797.026432:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776797.026439:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026444:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776797.026446:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026447:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.026449:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776797.026451:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026457:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776797.026460:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026467:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776797.026469:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026470:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776797.026472:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026473:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776797.026475:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776797.026478:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776797.026481:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776797.026483:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776797.026487:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776797.026494:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796224:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776797.026498:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026499:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.026501:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded500 x1862033099796096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026508:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776797.026510:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776797.026512:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776797.026514:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776797.026516:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776797.026517:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776797.026519:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776797.026521:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:2.0:1775776797.026523:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776797.026524:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776797.026525:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded500. 00000100:00000001:2.0:1775776797.026528:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776797.026529:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026530:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776797.026533:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee300 x1862033099796224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776822 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776797.026540:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776797.026541:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776797.026542:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776797.026545:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776797.026546:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776797.026547:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776797.026548:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776797.026550:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:2.0:1775776797.026552:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776797.026553:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776797.026554:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee300. 00000100:00000001:2.0:1775776797.026556:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776797.026557:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026559:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.026564:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776797.026565:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.026567:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776797.026569:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776797.723908:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776797.723916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776797.723918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776797.723923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776797.723927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776797.723931:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.723934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776797.723936:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776797.723939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776797.787874:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776797.787880:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776797.787886:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776797.787887:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776797.787890:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776797.787891:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776797.787894:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776797.787895:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776797.790464:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776797.790471:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776797.790476:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776797.852060:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776797.852068:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776797.852070:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776797.852076:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776797.852080:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776797.852083:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776797.852086:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776797.852088:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776797.852091:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776798.303816:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313186039, at 4313186052 00000100:00000010:0.0:1775776798.303833:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000040:1.0:1775776798.304064:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313186039, at 4313186052 00000100:00000010:1.0:1775776798.304069:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000001:2.0:1775776798.747916:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776798.747924:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776798.747926:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776798.747931:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776798.747934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776798.747938:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776798.747941:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776798.747944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776798.747947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776798.812034:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776798.812042:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776798.812044:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776798.812048:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776798.812052:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776798.812055:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776798.812056:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776798.812059:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776798.812063:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776798.812064:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776798.812068:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776798.812069:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776798.812072:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776798.812073:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776798.812168:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776798.812171:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776798.812174:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776798.875901:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776798.875909:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776798.875911:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776798.875918:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776798.875921:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776798.875926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776798.875929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776798.875931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776798.875934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776799.771893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776799.771899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776799.771901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776799.771906:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776799.771910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776799.771913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776799.771915:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776799.771918:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776799.771920:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776799.835947:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776799.835953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776799.835959:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776799.835960:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776799.835964:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776799.835965:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776799.835969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776799.835970:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776799.836158:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776799.836161:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776799.836164:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776799.899910:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776799.899917:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776799.899919:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776799.899924:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776799.899928:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776799.899932:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776799.899935:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776799.899937:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776799.899939:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776800.796530:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776800.796537:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776800.796539:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776800.796544:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776800.796548:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776800.796551:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776800.796554:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776800.796556:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776800.796558:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864820:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776800.864826:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864833:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776800.864836:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776800.864838:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776800.864841:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864844:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776800.864846:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864848:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864850:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864853:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776800.864854:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776800.864855:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776800.864856:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864858:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776800.864859:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864860:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864862:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864865:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776800.864866:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776800.864867:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776800.864869:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864870:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776800.864871:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864873:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776800.864875:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776800.865124:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776800.865127:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776800.865130:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776800.924344:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776800.924352:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776800.924354:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776800.924359:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776800.924362:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776800.924366:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776800.924369:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776800.924371:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776800.924374:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776801.820669:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776801.820676:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776801.820678:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776801.820684:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776801.820687:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776801.820691:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776801.820693:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776801.820696:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776801.820698:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776801.883913:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776801.883920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776801.883925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776801.883927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776801.883930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776801.883931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776801.883935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776801.883936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776801.884046:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776801.884049:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776801.884052:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776801.947890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776801.947896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776801.947898:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776801.947903:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776801.947906:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776801.947910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776801.947913:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776801.947915:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776801.947918:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776802.143273:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776802.143284:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776802.143287:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776802.143291:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776802.143296:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776802.143375:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18517 00000400:00000200:2.0:1775776802.143385:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776802.143392:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776802.143395:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776802.143399:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143402:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.143405:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143408:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776802.143412:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776802.143414:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776802.143416:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776802.143419:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776802.143421:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776802.143424:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776802.143427:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143429:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.143430:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143432:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776802.143435:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776802.143438:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143441:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776802.143443:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776802.143447:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776802.143449:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.143452:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776802.143454:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776802.143464:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c000. 00000020:00000040:2.0:1775776802.143467:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776802.143471:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776802.143472:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.143475:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776802.143478:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776802.143483:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776802.143491:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506800. 00000100:00000001:2.0:1775776802.143497:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776802.143506:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776802.143523:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776802.143527:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776802.143532:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4798067] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.143613:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776802.143617:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776802.143619:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776802.143625:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.143632:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776802.143634:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776802.143637:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776802.143639:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776802.143641:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776802.143644:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099796352:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776802.143648:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776802.143651:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776802.143654:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776802.143656:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776802.143661:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776802.143663:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776802.143669:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776802.143672:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776802.143679:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776802.143687:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099796352, portal 25 00000100:00000001:3.0:1775776802.143689:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776802.143691:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288710656 : -114841420840960 : ffff978d65e7c000) 00000100:00000040:3.0:1775776802.143696:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.143704:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776802.143707:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776802.143709:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776802.143713:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099796352, offset 0 00000400:00000200:3.0:1775776802.143719:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776802.143726:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776802.143732:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776802.143737:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776802.143741:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776802.143745:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776802.143748:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776802.143752:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4452:4452:256:4294967295] 192.168.201.139@tcp LPNI seq info [4452:4452:8:4294967295] 00000400:00000200:3.0:1775776802.143756:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776802.143766:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776802.143773:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776802.143842:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776802.143851:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776802.143854:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776802.143860:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776802.143863:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1775776802.143868:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776802.143870:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776802.143876:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776802.143881:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776802.143886:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776802.143889:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776802.143893:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776802.143897:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776802.143933:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776802.143937:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776802.143941:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776802.143942:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776802.143944:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776802.143947:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776802.143950:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18517 00000400:00000200:2.0:1775776802.143954:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776802.143957:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776802.143959:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776802.143962:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143964:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.143966:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143968:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776802.143971:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776802.143972:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776802.143974:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776802.143975:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776802.143977:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776802.143979:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776802.143982:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143984:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.143986:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143987:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776802.143990:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776802.143993:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776802.143995:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776802.143997:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776802.144001:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776802.144004:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.144006:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776802.144009:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776802.144013:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.144015:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776802.144018:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e300. 00000020:00000040:2.0:1775776802.144021:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776802.144023:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776802.144024:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.144026:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776802.144028:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776802.144029:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776802.144033:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504c00. 00000100:00000001:2.0:1775776802.144036:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776802.144053:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776802.144060:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776802.144063:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776802.144066:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff8c8494] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.147187:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776802.147200:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.147211:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776802.147213:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776802.147226:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776802.147232:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776802.147235:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313191161 00000100:00000010:3.0:1775776802.147243:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344900. 00000100:00000001:3.0:1775776802.147250:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776802.147257:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313191161 00000100:00000010:3.0:1775776802.147260:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344c60. 00000100:00000001:3.0:1775776802.147263:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776802.147266:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776802.147269:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776802.147272:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776802.147276:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776802/real 1775776802] req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776802.147287:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776802.147288:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776802.147291:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776802.147294:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776802.147477:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776802.147482:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776802.147486:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776802.147495:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.147503:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776802.147505:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.147507:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776802.147509:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776802.147513:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.147520:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776802.147524:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.147531:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776802.147534:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.147535:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776802.147536:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.147539:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776802.150468:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776802.150475:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776802.150485:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776802.150488:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776802.150495:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776802.150505:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099796352:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776802.150511:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.150514:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776802.150517:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c000 x1862033099796352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150524:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776802.150527:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776802.150531:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776802.150536:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776802.150539:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776802.150550:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776802.150553:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776802.150556:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506800. 02000000:00000001:3.0:1775776802.150561:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776802.150562:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776802.150564:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c000. 00000100:00000001:3.0:1775776802.150568:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776802.150569:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.150572:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.150580:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776802.150582:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776802.150585:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776802.150587:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776802.150589:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776802.150593:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150607:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776802.150608:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776802.150612:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776802.150615:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776802.150617:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776802.150620:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796480:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776802.150624:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776802.150628:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776802.150630:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776802.150633:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776802.150640:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776802.150642:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776802.150649:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776802.150654:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776802.150660:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776802.150667:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099796480, portal 10 00000100:00000001:3.0:1775776802.150669:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776802.150671:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288719616 : -114841420832000 : ffff978d65e7e300) 00000100:00000040:3.0:1775776802.150678:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150685:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776802.150688:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776802.150691:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776802.150694:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099796480, offset 0 00000400:00000200:3.0:1775776802.150700:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776802.150707:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776802.150715:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776802.150719:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776802.150723:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776802.150727:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776802.150729:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776802.150733:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4453:4453:256:4294967295] 192.168.201.139@tcp LPNI seq info [4453:4453:8:4294967295] 00000400:00000200:3.0:1775776802.150737:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776802.150747:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776802.150752:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776802.150758:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776802.150764:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776802.150767:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775776802.150772:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776802.150775:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1775776802.150803:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776802.150806:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776802.150811:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776802.150815:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776802.150820:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776802.150823:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776802.150826:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776802.150830:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150839:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776802.150842:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150847:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776802.150849:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776802.150853:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776802.150858:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776802.150861:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776802.150864:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313189896 00000100:00000001:3.0:1775776802.150866:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313191161 00000100:00000001:3.0:1775776802.150869:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313189896 00000100:00000001:3.0:1775776802.150871:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313191161 00000100:00000001:3.0:1775776802.150873:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776802.150876:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776802.150878:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776802.150882:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776802/real 1775776802] req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776802.150890:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776802.150892:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776802.150894:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776802.150896:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776802.150898:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776802.150900:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776802.150902:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776802.150911:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150917:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776802.150918:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.150920:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776802.150922:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776802.150925:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150931:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776802.150935:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150941:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776802.150943:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.150945:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776802.150946:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.150948:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776802.150950:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776802.150952:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776802.150956:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776802.150958:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776802.150962:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776802.150968:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796480:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776802.150972:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.150974:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776802.150977:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e300 x1862033099796480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776802.150982:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776802.150984:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776802.150986:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776802.150988:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776802.150989:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776802.150990:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776802.150993:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776802.150994:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:3.0:1775776802.150996:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776802.150998:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776802.150999:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e300. 00000100:00000001:3.0:1775776802.151001:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776802.151002:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.151004:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.151008:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776802.151009:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776802.151011:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776802.151012:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.151300:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776802.151302:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.151551:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776802.151557:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776802.151560:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776802.151562:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776802.151564:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776802.151567:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776802.151571:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18517 00000400:00000200:2.0:1775776802.151575:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776802.151578:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776802.151581:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776802.151584:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.151587:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.151589:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.151591:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776802.151594:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776802.151595:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776802.151597:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776802.151600:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776802.151602:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776802.151604:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776802.151607:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.151609:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.151611:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.151612:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776802.151615:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776802.151618:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776802.151620:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776802.151622:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776802.151627:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776802.151629:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.151632:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776802.151636:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776802.151638:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.151640:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776802.151645:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7fb80. 00000020:00000040:2.0:1775776802.151648:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776802.151651:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776802.151653:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.151656:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776802.151659:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776802.151662:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776802.151668:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505000. 00000100:00000001:2.0:1775776802.151673:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776802.151680:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776802.151689:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776802.151693:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776802.151697:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1787ebf] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.151740:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776802.151742:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776802.151744:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776802.151748:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.151755:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776802.151756:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776802.151758:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776802.151760:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776802.151762:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776802.151764:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796608:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776802.151768:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776802.151770:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776802.151771:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776802.151773:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776802.151802:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776802.151804:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776802.151809:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776802.151812:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776802.151817:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776802.151821:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099796608, portal 10 00000100:00000001:0.0:1775776802.151822:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776802.151824:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288725888 : -114841420825728 : ffff978d65e7fb80) 00000100:00000040:0.0:1775776802.151830:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.151837:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776802.151839:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776802.151841:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776802.151844:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099796608, offset 0 00000400:00000200:0.0:1775776802.151849:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776802.151853:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776802.151857:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776802.151860:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776802.151862:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776802.151864:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776802.151866:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776802.151868:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4454:4454:256:4294967295] 192.168.201.139@tcp LPNI seq info [4454:4454:8:4294967295] 00000400:00000200:0.0:1775776802.151872:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776802.151878:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776802.151882:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776802.151886:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776802.151890:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776802.151892:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776802.151896:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776802.151899:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776802.151902:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776802.151903:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776802.151907:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776802.151911:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776802.151914:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776802.151916:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776802.151918:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776802.151922:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776802.151966:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776802.151970:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776802.151974:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776802.151975:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776802.151977:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776802.151979:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776802.151982:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18517 00000400:00000200:2.0:1775776802.151986:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776802.151989:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776802.151991:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776802.151994:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.151996:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.151998:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.151999:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776802.152002:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776802.152003:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776802.152004:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776802.152006:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776802.152008:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776802.152010:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776802.152013:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.152015:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.152016:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.152018:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776802.152020:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776802.152022:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776802.152024:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776802.152026:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776802.152029:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776802.152031:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.152033:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776802.152035:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776802.152151:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.152153:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776802.152156:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e680. 00000020:00000040:2.0:1775776802.152158:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776802.152160:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776802.152161:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.152162:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776802.152163:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776802.152164:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776802.152167:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775776802.152169:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776802.152173:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776802.152178:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776802.152180:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776802.152182:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086433f13] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.154985:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776802.155000:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155011:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.155013:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776802.155026:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776802.155031:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776802.155252:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776802.155258:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313189908 00000100:00000001:0.0:1775776802.155262:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313191161 00000100:00000001:0.0:1775776802.155265:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313189908 00000100:00000001:0.0:1775776802.155267:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313191161 00000100:00000001:0.0:1775776802.155269:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776802.155272:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.155275:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776802.155280:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776802/real 1775776802] req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776802.155292:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776802.155294:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776802.155296:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776802.155300:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776802.155303:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776802.155306:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776802.155309:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776802.155322:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155329:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776802.155330:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155332:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776802.155334:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776802.155337:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155344:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776802.155348:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155356:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776802.155359:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155361:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776802.155362:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155364:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776802.155367:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776802.155370:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776802.155374:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776802.155376:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776802.155380:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776802.155389:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796608:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776802.155394:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155396:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776802.155399:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7fb80 x1862033099796608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155410:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776802.155413:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776802.155417:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776802.155421:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776802.155424:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776802.155425:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776802.155427:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776802.155429:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:0.0:1775776802.155432:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776802.155434:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776802.155436:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7fb80. 00000100:00000001:0.0:1775776802.155439:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776802.155440:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155443:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155450:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776802.155451:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.155454:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776802.155457:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776802.155458:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776802.155462:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155470:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776802.155472:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776802.155475:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776802.155478:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776802.155481:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776802.155484:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796736:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776802.155487:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776802.155491:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776802.155493:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776802.155495:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776802.155501:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776802.155503:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776802.155508:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776802.155511:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776802.155516:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776802.155521:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099796736, portal 4 00000100:00000001:0.0:1775776802.155523:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776802.155525:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288720512 : -114841420831104 : ffff978d65e7e680) 00000100:00000040:0.0:1775776802.155531:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155538:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776802.155572:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776802.155575:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776802.155578:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099796736, offset 0 00000400:00000200:0.0:1775776802.155584:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776802.155591:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776802.155597:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776802.155601:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776802.155605:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776802.155608:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776802.155611:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776802.155614:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4455:4455:256:4294967295] 192.168.201.139@tcp LPNI seq info [4455:4455:8:4294967295] 00000400:00000200:0.0:1775776802.155619:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776802.155627:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776802.155633:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776802.155638:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776802.155644:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776802.155648:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776802.155652:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776802.155655:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776802.155659:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776802.155661:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776802.155666:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776802.155670:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776802.155674:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776802.155676:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776802.155679:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776802.155683:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155692:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776802.155694:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155701:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.155702:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776802.155706:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776802.155709:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776802.155711:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776802.155713:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313189917 00000100:00000001:0.0:1775776802.155716:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313191161 00000100:00000001:0.0:1775776802.155718:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313189917 00000100:00000001:0.0:1775776802.155720:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313191161 00000100:00000001:0.0:1775776802.155722:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776802.155724:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.155726:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776802.155729:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776802/real 1775776802] req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776802.155736:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776802.155737:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776802.155738:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776802.155741:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776802.155742:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776802.155744:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776802.155747:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776802.155754:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155761:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776802.155763:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155765:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776802.155766:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776802.155769:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155776:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776802.155909:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155917:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776802.155919:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155921:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776802.155922:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155924:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776802.155926:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776802.155929:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776802.155932:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776802.155934:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776802.155938:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776802.155947:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796736:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776802.155950:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155952:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776802.155954:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e680 x1862033099796736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.155960:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776802.155962:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776802.155964:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776802.155967:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776802.155968:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776802.155970:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776802.155972:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776802.155973:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:0.0:1775776802.155976:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776802.155977:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776802.155978:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e680. 00000100:00000001:0.0:1775776802.155980:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776802.155982:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155983:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.155987:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776802.155988:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.155989:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776802.155991:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.156704:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776802.156710:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776802.156714:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776802.156716:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776802.156718:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776802.156722:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776802.156726:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18517 00000400:00000200:2.0:1775776802.156733:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776802.156737:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776802.156740:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776802.156743:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.156746:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.156749:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.156751:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776802.156754:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776802.156755:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776802.156757:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776802.156759:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776802.156762:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776802.156764:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776802.156766:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.156768:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776802.156770:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776802.156771:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776802.156774:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776802.156776:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776802.156807:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776802.156809:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776802.156815:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776802.156818:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.156820:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776802.156824:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776802.156826:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.156828:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776802.156834:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ea00. 00000020:00000040:2.0:1775776802.156838:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776802.156841:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776802.156842:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776802.156845:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776802.156847:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776802.156851:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776802.156857:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:2.0:1775776802.156863:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776802.156869:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776802.156877:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776802.156882:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776802.156886:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc493d27] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776802.156895:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776802.156900:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18527) 00000100:00000001:0.0:1775776802.157018:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776802.157021:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776802.157023:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776802.157028:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.157035:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776802.157036:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776802.157039:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776802.157051:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776802.157053:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776802.157056:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796864:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776802.157060:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776802.157062:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776802.157064:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776802.157066:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776802.157070:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776802.157071:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776802.157076:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776802.157079:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776802.157084:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776802.157089:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099796864, portal 4 00000100:00000001:0.0:1775776802.157091:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776802.157092:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288721408 : -114841420830208 : ffff978d65e7ea00) 00000100:00000040:0.0:1775776802.157099:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.157105:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776802.157106:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776802.157109:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776802.157111:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099796864, offset 0 00000400:00000200:0.0:1775776802.157115:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776802.157121:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776802.157124:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776802.157128:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776802.157130:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776802.157134:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776802.157136:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776802.157139:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4456:4456:256:4294967295] 192.168.201.139@tcp LPNI seq info [4456:4456:8:4294967295] 00000400:00000200:0.0:1775776802.157143:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776802.157150:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776802.157154:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776802.157158:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776802.157162:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776802.157165:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776802.157170:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776802.157172:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776802.157176:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776802.157177:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776802.157181:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776802.157185:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776802.157188:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776802.157190:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776802.157193:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776802.157196:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.157205:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776802.157208:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776827 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.157214:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.157215:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776802.157219:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776802.157223:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776802.157226:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776802.157228:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313189910 00000100:00000001:0.0:1775776802.157230:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313191161 00000100:00000001:0.0:1775776802.157232:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313189910 00000100:00000001:0.0:1775776802.157234:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313191161 00000100:00000001:0.0:1775776802.157236:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776802.157238:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.157240:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776802.157244:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776802/real 1775776802] req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776802.157251:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776802.157252:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776802.157254:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776802.157256:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776802.157258:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776802.157260:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776802.157264:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776802.157272:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.157278:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776802.157280:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.157282:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776802.157283:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776802.157287:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.157293:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776802.157296:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.157302:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776802.157305:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.157306:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776802.157308:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.157309:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776802.157311:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776802.157314:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776802.157317:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776802.157320:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776802.157323:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776802.157329:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099796864:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776802.157333:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.157335:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776802.157337:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ea00 x1862033099796864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776827 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776802.157344:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776802.157345:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776802.157348:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776802.157351:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776802.157352:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776802.157354:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776802.157356:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776802.157357:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:0.0:1775776802.157360:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776802.157361:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776802.157362:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ea00. 00000100:00000001:0.0:1775776802.157365:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776802.157366:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.157368:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776802.157372:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776802.157373:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776802.157374:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776802.157376:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776802.844301:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776802.844308:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776802.844310:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776802.844316:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776802.844320:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776802.844323:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.844327:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776802.844329:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776802.844332:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776802.907915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776802.907921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776802.907926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776802.907927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776802.907930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776802.907931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776802.907934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776802.907935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776802.908334:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776802.908338:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776802.908341:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776802.972486:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776802.972494:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776802.972496:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776802.972501:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776802.972505:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776802.972509:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776802.972511:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776802.972514:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776802.972517:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776803.419918:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313191161, at 4313191168 00000100:00000010:0.0:1775776803.419937:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344900. 00000100:00000040:1.0:1775776803.419948:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313191161, at 4313191168 00000100:00000010:1.0:1775776803.420169:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c60. 00000100:00000001:3.0:1775776803.867902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776803.867909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776803.867911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776803.867916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776803.867920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776803.867924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776803.867926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776803.867929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776803.867932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776803.931987:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776803.931995:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776803.932008:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776803.932010:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776803.932012:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776803.932013:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776803.932016:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776803.932016:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776803.932067:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776803.932070:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776803.932074:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776803.995905:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776803.995912:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776803.995914:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776803.995921:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776803.995925:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776803.995929:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776803.995931:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776803.995934:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776803.995937:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776804.891938:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776804.891945:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776804.891947:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776804.891952:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776804.891955:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776804.891975:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776804.891978:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776804.891980:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776804.891983:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776804.955942:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776804.955947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776804.955951:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776804.955952:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776804.955955:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776804.955955:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776804.955958:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776804.955959:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776804.956071:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776804.956075:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776804.956078:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776805.021718:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776805.021726:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776805.021727:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776805.021732:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776805.021736:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776805.021739:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776805.021742:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776805.021744:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776805.021747:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776805.915904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776805.915912:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776805.915914:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776805.915919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776805.915922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776805.915927:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776805.915929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776805.915932:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776805.915934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776805.979906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776805.979914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776805.979920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776805.979921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776805.979925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776805.979926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776805.979929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776805.979931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776805.980037:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776805.980040:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776805.980043:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776806.044717:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776806.044724:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776806.044727:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776806.044732:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776806.044736:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776806.044740:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776806.044742:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776806.044744:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776806.044747:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776806.939893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776806.939898:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776806.939899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776806.939903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776806.939906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776806.939910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776806.939912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776806.939914:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776806.939917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776807.004150:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776807.004157:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776807.004163:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776807.004165:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776807.004168:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776807.004170:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776807.004172:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776807.005120:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776807.005665:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776807.005669:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776807.005672:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776807.068743:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776807.068749:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776807.068751:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776807.068757:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776807.068761:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776807.068764:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776807.068767:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776807.068770:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776807.068773:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776807.259921:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776807.259930:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776807.259933:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776807.259937:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776807.259942:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776807.259949:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18522 00000400:00000200:2.0:1775776807.259959:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776807.259965:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776807.259968:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776807.259971:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.259974:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.259977:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.259979:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776807.259983:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776807.259985:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776807.259987:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776807.259990:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776807.259992:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776807.259995:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776807.259997:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.259999:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.260001:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.260003:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776807.260006:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776807.260009:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776807.260011:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776807.260014:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776807.260017:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776807.260020:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.260022:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776807.260024:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776807.260034:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ce00. 00000020:00000040:2.0:1775776807.260037:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776807.260040:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776807.260042:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.260045:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776807.260047:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776807.260051:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776807.260058:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:2.0:1775776807.260063:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776807.260072:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776807.260088:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776807.260092:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776807.260096:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050a5f9f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260169:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.260173:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776807.260175:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776807.260180:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260188:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776807.260190:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.260194:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776807.260196:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776807.260199:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776807.260201:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099796992:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776807.260205:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776807.260209:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776807.260211:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.260214:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776807.260219:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.260220:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776807.260226:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776807.260230:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776807.260236:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776807.260242:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099796992, portal 25 00000100:00000001:0.0:1775776807.260244:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776807.260246:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288714240 : -114841420837376 : ffff978d65e7ce00) 00000100:00000040:0.0:1775776807.260251:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260258:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776807.260261:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776807.260263:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776807.260266:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099796992, offset 0 00000400:00000200:0.0:1775776807.260272:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776807.260280:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.260286:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776807.260291:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776807.260295:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776807.260298:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776807.260301:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776807.260305:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4457:4457:256:4294967295] 192.168.201.139@tcp LPNI seq info [4457:4457:8:4294967295] 00000400:00000200:0.0:1775776807.260309:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.260319:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776807.260324:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.260328:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776807.260335:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776807.260338:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775776807.260343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776807.260346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775776807.260350:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.260353:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776807.260359:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776807.260363:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776807.260368:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776807.260371:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776807.260375:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776807.260379:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776807.260404:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776807.260407:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776807.260411:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776807.260412:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776807.260414:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776807.260417:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776807.260419:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18522 00000400:00000200:2.0:1775776807.260422:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776807.260425:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776807.260427:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776807.260430:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.260432:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.260434:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.260436:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776807.260438:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776807.260439:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776807.260441:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776807.260443:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776807.260444:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776807.260446:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776807.260449:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.260463:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.260465:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.260467:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776807.260470:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776807.260472:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776807.260475:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776807.260476:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776807.260481:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776807.260483:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.260485:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776807.260489:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776807.260491:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.260493:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776807.260497:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d180. 00000020:00000040:2.0:1775776807.260499:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776807.260501:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776807.260503:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.260504:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776807.260506:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776807.260507:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776807.260511:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506c00. 00000100:00000001:2.0:1775776807.260514:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776807.260518:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776807.260525:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776807.260527:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776807.260531:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008292136e] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260559:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.260563:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260570:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.260571:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776807.260578:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776807.260582:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776807.260584:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313196277 00000100:00000010:0.0:1775776807.260589:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237120. 00000100:00000001:0.0:1775776807.260595:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776807.260601:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313196277 00000100:00000010:0.0:1775776807.260604:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237660. 00000100:00000001:0.0:1775776807.260607:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776807.260610:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776807.260612:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.260615:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776807.260618:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776807/real 1775776807] req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776807.260627:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776807.260629:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776807.260631:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776807.260634:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776807.260635:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776807.260637:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776807.260641:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776807.260649:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260656:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776807.260658:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.260660:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.260662:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776807.260666:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260672:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776807.260676:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260684:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776807.260687:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.260688:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.260689:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.260691:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776807.260693:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776807.260696:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776807.260699:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776807.260701:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776807.260705:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776807.260712:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099796992:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776807.260717:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.260718:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.260721:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ce00 x1862033099796992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260728:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776807.260730:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776807.260732:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.260735:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776807.260737:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776807.260738:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776807.260740:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776807.260743:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:0.0:1775776807.260746:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776807.260748:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776807.260750:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ce00. 00000100:00000001:0.0:1775776807.260753:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776807.260754:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.260756:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.260762:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776807.260764:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.260766:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.260767:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776807.260768:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776807.260771:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260825:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776807.260826:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.260829:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776807.260830:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776807.260832:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776807.260835:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797120:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776807.260838:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776807.260840:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776807.260841:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.260843:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776807.260847:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.260848:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776807.260851:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776807.260854:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776807.260857:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776807.260860:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099797120, portal 10 00000100:00000001:0.0:1775776807.260862:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776807.260863:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288715136 : -114841420836480 : ffff978d65e7d180) 00000100:00000040:0.0:1775776807.260869:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260875:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776807.260877:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776807.260879:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776807.260881:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099797120, offset 0 00000400:00000200:0.0:1775776807.260885:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776807.260889:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.260892:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776807.260895:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776807.260898:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776807.260900:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776807.260902:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776807.260905:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4458:4458:256:4294967295] 192.168.201.139@tcp LPNI seq info [4458:4458:8:4294967295] 00000400:00000200:0.0:1775776807.260909:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.260916:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776807.260921:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.260924:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776807.260927:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776807.260930:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775776807.260934:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776807.260936:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775776807.260940:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.260942:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776807.260945:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776807.260948:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776807.260951:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776807.260953:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776807.260955:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776807.260958:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260971:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.260973:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.260980:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.260981:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776807.260984:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776807.260987:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776807.260989:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776807.260991:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313195012 00000100:00000001:0.0:1775776807.260994:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313196277 00000100:00000001:0.0:1775776807.260996:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313195012 00000100:00000001:0.0:1775776807.260998:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313196277 00000100:00000001:0.0:1775776807.260999:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776807.261001:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.261002:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776807.261005:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776807/real 1775776807] req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776807.261013:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776807.261014:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776807.261015:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776807.261017:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776807.261019:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776807.261021:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776807.261023:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776807.261031:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.261038:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776807.261040:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.261041:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.261043:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776807.261046:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.261053:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776807.261056:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.261063:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776807.261065:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.261067:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.261068:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.261070:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776807.261072:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776807.261074:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776807.261077:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776807.261079:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776807.261082:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776807.261090:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797120:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776807.261093:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.261094:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.261097:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d180 x1862033099797120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.261103:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776807.261104:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776807.261106:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.261108:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776807.261109:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776807.261110:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776807.261112:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776807.261114:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:0.0:1775776807.261116:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776807.261117:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776807.261119:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d180. 00000100:00000001:0.0:1775776807.261121:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776807.261122:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.261124:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.261126:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776807.261128:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.261130:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.261131:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.261840:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.261843:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.261888:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776807.261893:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776807.261897:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776807.261899:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776807.261901:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776807.261905:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776807.261910:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18522 00000400:00000200:2.0:1775776807.261917:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776807.261922:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776807.261924:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776807.261928:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.261930:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.261933:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.261935:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776807.261938:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776807.261939:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776807.261941:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776807.261943:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776807.261945:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776807.261947:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776807.261950:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.261951:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.261953:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.261954:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776807.261957:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776807.261959:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776807.261961:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776807.261963:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776807.261969:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776807.261972:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.261974:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776807.261977:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776807.261980:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.261982:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776807.261988:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ca80. 00000020:00000040:2.0:1775776807.261991:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776807.261994:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776807.261996:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.261998:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776807.262000:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776807.262003:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776807.262008:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506000. 00000100:00000001:2.0:1775776807.262013:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776807.262020:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776807.262029:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776807.262032:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776807.262036:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a559b88] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262247:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.262250:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776807.262252:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776807.262258:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262266:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776807.262267:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.262270:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776807.262273:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776807.262275:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776807.262278:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797248:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776807.262282:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776807.262285:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776807.262287:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.262289:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776807.262294:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.262296:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776807.262301:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776807.262306:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776807.262312:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776807.262318:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099797248, portal 10 00000100:00000001:0.0:1775776807.262321:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776807.262322:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288713344 : -114841420838272 : ffff978d65e7ca80) 00000100:00000040:0.0:1775776807.262328:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262335:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776807.262338:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776807.262340:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776807.262343:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099797248, offset 0 00000400:00000200:0.0:1775776807.262348:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776807.262353:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.262359:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776807.262363:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776807.262366:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776807.262369:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776807.262372:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776807.262375:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4459:4459:256:4294967295] 192.168.201.139@tcp LPNI seq info [4459:4459:8:4294967295] 00000400:00000200:0.0:1775776807.262379:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.262388:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776807.262393:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.262396:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776807.262401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776807.262404:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775776807.262409:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776807.262411:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775776807.262414:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.262416:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776807.262421:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776807.262425:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776807.262428:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776807.262430:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776807.262434:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776807.262437:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776807.262474:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776807.262477:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776807.262480:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776807.262482:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776807.262484:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776807.262486:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776807.262489:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18522 00000400:00000200:2.0:1775776807.262493:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776807.262496:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776807.262498:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776807.262501:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.262503:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.262505:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.262507:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776807.262509:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776807.262510:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776807.262512:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776807.262513:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776807.262515:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776807.262518:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776807.262521:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.262523:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.262524:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.262526:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776807.262528:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776807.262531:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776807.262533:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776807.262534:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776807.262537:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776807.262539:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.262541:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776807.262543:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776807.262545:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.262547:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776807.262551:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d180. 00000020:00000040:2.0:1775776807.262553:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776807.262555:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776807.262557:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.262558:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776807.262560:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776807.262562:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776807.262565:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775776807.262568:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776807.262573:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776807.262580:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776807.262582:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776807.262586:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008292136e] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262611:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.262615:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262622:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.262624:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776807.262628:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776807.262632:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776807.262635:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776807.262638:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313195015 00000100:00000001:0.0:1775776807.262640:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313196277 00000100:00000001:0.0:1775776807.262642:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313195015 00000100:00000001:0.0:1775776807.262645:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313196277 00000100:00000001:0.0:1775776807.262647:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776807.262649:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.262651:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776807.262654:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776807/real 1775776807] req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776807.262662:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776807.262664:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776807.262666:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776807.262668:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776807.262670:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776807.262672:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776807.262675:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776807.262683:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262689:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776807.262691:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.262693:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.262695:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776807.262699:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262706:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776807.262709:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262716:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776807.262718:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.262720:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.262721:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.262723:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776807.262725:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776807.262728:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776807.262731:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776807.262733:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776807.262736:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776807.262745:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797248:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776807.262749:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.262751:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.262753:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ca80 x1862033099797248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262759:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776807.262761:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776807.262763:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.262766:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776807.262768:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776807.262769:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776807.262771:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776807.262773:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506000. 02000000:00000001:0.0:1775776807.262776:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776807.262808:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776807.262810:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ca80. 00000100:00000001:0.0:1775776807.262813:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776807.262814:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.262816:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.262822:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776807.262823:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.262826:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.262827:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776807.262828:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776807.262832:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262838:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776807.262839:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.262841:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776807.262843:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776807.262845:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776807.262848:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797376:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776807.262851:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776807.262852:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776807.262854:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.262855:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776807.262858:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.262860:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776807.262864:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776807.262866:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776807.262869:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776807.262871:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099797376, portal 4 00000100:00000001:0.0:1775776807.262873:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776807.262875:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288715136 : -114841420836480 : ffff978d65e7d180) 00000100:00000040:0.0:1775776807.262880:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262886:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776807.262889:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776807.262891:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776807.262893:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099797376, offset 0 00000400:00000200:0.0:1775776807.262896:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776807.262900:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.262903:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776807.262906:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776807.262908:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776807.262911:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776807.262913:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776807.262916:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4460:4460:256:4294967295] 192.168.201.139@tcp LPNI seq info [4460:4460:8:4294967295] 00000400:00000200:0.0:1775776807.262920:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.262926:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776807.262930:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.262933:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776807.262937:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776807.262940:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775776807.262944:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776807.262946:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775776807.262964:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.262966:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776807.262969:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776807.262972:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776807.262975:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776807.262977:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776807.262979:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776807.262983:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.262991:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.262993:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.263005:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.263007:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776807.263010:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776807.263012:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776807.263015:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776807.263017:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313195024 00000100:00000001:0.0:1775776807.263020:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313196277 00000100:00000001:0.0:1775776807.263022:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313195024 00000100:00000001:0.0:1775776807.263024:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313196277 00000100:00000001:0.0:1775776807.263026:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776807.263028:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.263029:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776807.263032:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776807/real 1775776807] req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776807.263039:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776807.263040:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776807.263042:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776807.263044:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776807.263045:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776807.263047:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776807.263050:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776807.263058:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.263063:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776807.263065:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.263066:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.263068:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776807.263071:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.263078:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776807.263082:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.263088:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776807.263090:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.263092:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.263093:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.263094:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776807.263096:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776807.263098:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776807.263100:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776807.263102:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776807.263105:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776807.263113:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797376:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776807.263116:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.263118:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.263120:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d180 x1862033099797376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.263127:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776807.263128:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776807.263130:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.263132:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776807.263133:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776807.263134:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776807.263136:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776807.263138:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:0.0:1775776807.263140:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776807.263142:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776807.263143:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d180. 00000100:00000001:0.0:1775776807.263145:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776807.263147:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.263148:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.263151:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776807.263152:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.263154:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.263156:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.263638:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.263641:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.263674:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776807.263678:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776807.263682:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776807.263684:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776807.263686:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776807.263688:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776807.263692:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18522 00000400:00000200:2.0:1775776807.263697:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776807.263700:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776807.263703:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776807.263706:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.263708:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.263710:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.263713:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776807.263715:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776807.263716:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776807.263718:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776807.263720:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776807.263722:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776807.263725:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776807.263727:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.263729:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776807.263731:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776807.263733:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776807.263735:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776807.263738:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776807.263740:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776807.263742:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776807.263745:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776807.263747:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.263749:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776807.263752:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776807.263755:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.263756:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776807.263761:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ce00. 00000020:00000040:2.0:1775776807.263763:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776807.263766:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776807.263767:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776807.263769:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776807.263772:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776807.263774:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776807.263793:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775776807.263797:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776807.263802:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776807.263810:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776807.263813:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776807.263817:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050a5f9f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.263853:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.263855:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776807.263856:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776807.263860:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.263868:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776807.263870:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.263872:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776807.263874:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776807.263876:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776807.263879:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797504:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776807.263882:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776807.263884:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776807.263885:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776807.263887:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776807.263890:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.263892:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776807.263895:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776807.263898:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776807.263901:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776807.263905:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099797504, portal 4 00000100:00000001:0.0:1775776807.263906:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776807.263908:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288714240 : -114841420837376 : ffff978d65e7ce00) 00000100:00000040:0.0:1775776807.263913:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.263920:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776807.263923:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776807.263924:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776807.263927:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099797504, offset 0 00000400:00000200:0.0:1775776807.263931:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776807.263934:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.263938:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776807.263941:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776807.263943:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776807.263946:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776807.263948:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776807.263951:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4461:4461:256:4294967295] 192.168.201.139@tcp LPNI seq info [4461:4461:8:4294967295] 00000400:00000200:0.0:1775776807.263955:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776807.263963:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776807.263968:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.263971:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776807.263975:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776807.263978:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775776807.263983:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776807.263985:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775776807.263989:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776807.263991:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776807.263994:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776807.263997:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776807.264000:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776807.264002:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776807.264005:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776807.264008:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776807.264031:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776807.264036:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18532) 00000100:00000001:0.0:1775776807.264068:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.264072:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776832 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.264085:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.264087:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776807.264091:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776807.264094:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776807.264097:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776807.264099:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313195017 00000100:00000001:0.0:1775776807.264101:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313196277 00000100:00000001:0.0:1775776807.264103:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313195017 00000100:00000001:0.0:1775776807.264105:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313196277 00000100:00000001:0.0:1775776807.264107:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776807.264109:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.264111:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776807.264114:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776807/real 1775776807] req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776807.264122:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776807.264123:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776807.264125:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776807.264127:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776807.264129:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776807.264131:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776807.264134:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776807.264142:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.264149:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776807.264150:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.264152:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.264153:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776807.264156:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.264163:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776807.264167:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.264174:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776807.264176:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.264178:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776807.264179:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.264180:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776807.264182:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776807.264184:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776807.264187:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776807.264189:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776807.264193:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776807.264200:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797504:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776807.264204:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.264206:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776807.264209:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ce00 x1862033099797504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776832 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776807.264214:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776807.264216:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776807.264218:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776807.264220:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776807.264222:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776807.264223:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776807.264225:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776807.264226:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:0.0:1775776807.264229:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776807.264231:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776807.264233:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ce00. 00000100:00000001:0.0:1775776807.264235:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776807.264237:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.264239:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776807.264243:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776807.264244:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776807.264246:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776807.264247:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776807.964270:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776807.964277:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776807.964278:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776807.964283:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776807.964286:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776807.964291:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776807.964293:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776807.964295:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776807.964298:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776808.030550:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776808.030558:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776808.030564:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776808.030565:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776808.030568:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776808.030570:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776808.030572:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776808.030573:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776808.030675:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776808.030678:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776808.030681:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776808.091904:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776808.091912:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776808.091913:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776808.091919:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776808.091922:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776808.091926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776808.091929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776808.091931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776808.091934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776808.539978:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313196277, at 4313196288 00000100:00000010:0.0:1775776808.539993:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237120. 00000100:00000040:1.0:1775776808.539999:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313196277, at 4313196288 00000100:00000010:1.0:1775776808.540008:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237660. 00000100:00000001:3.0:1775776808.987916:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776808.987923:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776808.987925:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776808.987930:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776808.987934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776808.987938:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776808.987940:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776808.987943:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776808.987947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776809.052253:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776809.052261:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776809.052264:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776809.052267:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776809.052271:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776809.052273:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776809.052275:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776809.052278:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776809.052283:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776809.052284:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776809.052287:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776809.052288:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776809.052290:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776809.052291:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776809.052489:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776809.052493:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776809.052496:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776809.115882:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776809.115887:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776809.115888:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776809.115892:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776809.115895:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776809.115898:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776809.115900:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776809.115902:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776809.115904:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776810.011866:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776810.011873:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776810.011875:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776810.011879:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776810.011883:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776810.011886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776810.011888:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776810.011889:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776810.011891:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776810.075999:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776810.076021:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776810.076028:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776810.076029:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776810.076032:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776810.076033:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776810.076036:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776810.076037:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776810.076308:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776810.076312:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776810.076315:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776810.139886:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776810.139893:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776810.139896:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776810.139901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776810.139905:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776810.139909:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776810.139911:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776810.139913:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776810.139916:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776811.035889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776811.035895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776811.035896:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776811.035900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776811.035903:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776811.035906:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776811.035908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776811.035909:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776811.035911:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099894:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776811.099900:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099913:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776811.099916:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776811.099918:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776811.099922:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099925:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776811.099927:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099929:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099931:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776811.099935:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776811.099936:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776811.099937:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099939:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776811.099940:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099941:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099943:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099946:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776811.099947:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776811.099948:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776811.099950:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099951:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776811.099952:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099953:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776811.099955:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776811.100023:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776811.100027:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776811.100029:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776811.163855:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776811.163862:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776811.163863:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776811.163869:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776811.163872:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776811.163876:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776811.163878:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776811.163880:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776811.163883:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776812.059925:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776812.059933:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776812.059935:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776812.059941:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776812.059946:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776812.059950:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776812.059953:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776812.059956:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776812.059959:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776812.124074:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776812.124080:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776812.124087:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776812.124088:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776812.124091:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776812.124092:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776812.124095:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776812.124096:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776812.124481:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776812.124484:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776812.124487:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776812.187900:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776812.187907:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776812.187909:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776812.187914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776812.187918:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776812.187922:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776812.187925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776812.187927:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776812.187930:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776812.380384:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776812.380393:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776812.380396:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776812.380400:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776812.380404:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776812.380409:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18527 00000400:00000200:0.0:1775776812.380420:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776812.380425:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776812.380428:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776812.380431:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380434:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776812.380436:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380439:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776812.380442:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776812.380444:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776812.380448:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776812.380450:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776812.380452:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776812.380455:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776812.380458:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380460:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776812.380461:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380463:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776812.380466:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776812.380469:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380471:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776812.380474:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776812.380476:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776812.380479:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.380481:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776812.380483:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776812.380493:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776812.380496:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776812.380500:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776812.380502:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.380505:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776812.380507:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776812.380511:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776812.380518:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776812.380524:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776812.380532:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776812.380542:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776812.380546:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776812.380551:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776812.380580:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776812.380584:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776812.380587:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776812.380588:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776812.380590:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776812.380592:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776812.380595:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18527 00000400:00000200:0.0:1775776812.380598:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776812.380601:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776812.380603:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776812.380606:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380609:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776812.380612:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380614:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776812.380616:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776812.380618:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776812.380619:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776812.380622:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000100:00000001:3.0:1775776812.380622:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775776812.380625:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776812.380627:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775776812.380629:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:3.0:1775776812.380630:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776812.380633:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380636:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776812.380639:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776812.380640:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380643:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776812.380648:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776812.380653:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380658:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.380658:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1775776812.380661:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775776812.380661:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.380665:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776812.380667:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776812.380667:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776812.380672:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.380673:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.380675:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:3.0:1775776812.380679:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099797632:192.168.201.139@tcp:250:kworker.0 00000008:00100000:0.0:1775776812.380682:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1775776812.380683:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1775776812.380687:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.380688:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776812.380689:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.380691:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776812.380694:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 02000000:00000001:3.0:1775776812.380694:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1775776812.380698:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776812.380701:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:3.0:1775776812.380701:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115bc00. 02000000:00000001:0.0:1775776812.380703:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.380705:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.380706:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776812.380709:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776812.380711:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776812.380715:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000010:0.0:1775776812.380717:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000400:00000010:3.0:1775776812.380720:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000100:00000001:0.0:1775776812.380722:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1775776812.380727:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00100000:0.0:1775776812.380730:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1775776812.380735:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099797632, portal 25 00000100:00000001:3.0:1775776812.380739:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776812.380741:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776812.380742:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741760640 : -114841967790976 : ffff978d454df480) 00000100:00000001:0.0:1775776812.380745:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776812.380748:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776812.380757:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776812.380762:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776812.380767:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775776812.380772:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1775776812.380774:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:3.0:1775776812.380802:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776812.380805:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776812.380809:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099797632, offset 0 00000100:00000001:0.0:1775776812.380811:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776812.380814:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776812.380818:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:3.0:1775776812.380818:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00080000:0.0:1775776812.380823:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18527 00000400:00000200:0.0:1775776812.380830:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.380830:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776812.380836:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776812.380840:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1775776812.380840:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000040:0.0:1775776812.380847:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776812.380848:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775776812.380852:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1775776812.380855:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000040:0.0:1775776812.380858:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776812.380861:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776812.380864:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776812.380868:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4462:4462:256:4294967295] 192.168.201.139@tcp LPNI seq info [4462:4462:8:4294967295] 00000100:00000001:0.0:1775776812.380874:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:3.0:1775776812.380876:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:0.0:1775776812.380878:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776812.380880:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776812.380882:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776812.380885:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776812.380890:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:3.0:1775776812.380890:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000020:00000001:0.0:1775776812.380894:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776812.380899:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:3.0:1775776812.380902:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776812.380903:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776812.380907:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000010:3.0:1775776812.380908:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000100:00000001:0.0:1775776812.380910:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776812.380913:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776812.380918:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000800:00000200:3.0:1775776812.380918:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776812.380924:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775776812.380924:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 02000000:00000001:0.0:1775776812.380927:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776812.380932:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.1:1775776812.380933:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776812.380935:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.380938:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:3.1:1775776812.380938:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000008:00100000:0.0:1775776812.380941:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775776812.380944:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000008:00000001:0.0:1775776812.380947:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1775776812.380948:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000100:00000001:0.0:1775776812.380950:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776812.380955:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:0.0:1775776812.380956:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776812.380960:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000400:00000200:3.0:1775776812.380963:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1775776812.380964:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776812.380967:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776812.380968:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000100:00000001:0.0:1775776812.380969:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776812.380971:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1775776812.380972:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 02000000:00000001:0.0:1775776812.380974:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.380977:0:301888:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:0.0:1775776812.380979:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000200:3.0:1775776812.380983:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776812.380984:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.380996:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.381000:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776812.381003:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776812.381008:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381010:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776812.381016:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.381020:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1775776812.381023:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776812.381025:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381027:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313201398 00000100:00000040:0.0:1775776812.381031:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775776812.381032:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344360. 00000100:00000001:0.0:1775776812.381041:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381041:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1775776812.381045:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776812.381049:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776812.381049:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313201398 00000100:00000001:0.0:1775776812.381050:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776812.381053:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:3.0:1775776812.381054:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503441e0. 00000100:00000001:0.0:1775776812.381057:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776812.381059:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1775776812.381061:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18527 00000100:00000001:3.0:1775776812.381064:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775776812.381068:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776812.381068:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381072:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000400:00000200:0.0:1775776812.381073:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776812.381076:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776812.381078:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.381080:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776812.381082:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.381084:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776812.381085:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776812.381086:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776812.381088:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776812.381090:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776812.381093:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00100000:3.0:1775776812.381093:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776812/real 1775776812] req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776812.381097:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776812.381104:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776812.381108:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776812.381111:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000001:3.0:1775776812.381112:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1775776812.381115:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000001:3.0:1775776812.381115:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776812.381118:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000020:00000040:0.0:1775776812.381119:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000400:00000200:3.0:1775776812.381123:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000100:00080000:0.0:1775776812.381124:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000010:3.0:1775776812.381127:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:0.0:1775776812.381128:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381130:0:301888:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1775776812.381131:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776812.381135:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1775776812.381136:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776812.381138:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775776812.381148:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776812.381151:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776812.381155:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1775776812.381158:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000001:0.0:1775776812.381160:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381160:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776812.381164:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381165:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.381167:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1775776812.381170:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000100:00100000:3.0:1775776812.381172:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776812.381173:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776812.381178:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776812.381181:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.381184:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776812.381186:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776812.381188:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381188:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1775776812.381193:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000040:3.0:1775776812.381194:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776812.381198:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381206:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:0.0:1775776812.381207:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776812.381211:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381215:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.381217:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381222:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776812.381223:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776812.381225:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776812.381226:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776812.381229:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1775776812.381232:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776812.381235:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776812.381239:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776812.381246:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776812.381247:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776812.381253:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776812.381259:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776812.381261:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776812.381264:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775776812.381265:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099797632:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776812.381267:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776812.381272:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18527 00000100:00000001:3.0:1775776812.381273:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381276:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1775776812.381279:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775776812.381281:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df480 x1862033099797632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776812.381284:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776812.381289:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776812.381294:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776812.381295:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1775776812.381298:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1775776812.381298:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:0.0:1775776812.381301:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000010:3.0:1775776812.381302:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115bc00. 00000100:00000001:0.0:1775776812.381304:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:3.0:1775776812.381306:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776812.381307:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000001:3.0:1775776812.381308:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776812.381309:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776812.381311:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776812.381311:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000040:0.0:1775776812.381314:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776812.381315:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:0.0:1775776812.381317:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 02000000:00000010:3.0:1775776812.381318:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 00000020:00000001:0.0:1775776812.381321:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 02000000:00000001:3.0:1775776812.381324:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:0.0:1775776812.381325:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000001:3.0:1775776812.381326:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1775776812.381328:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:3.0:1775776812.381329:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df480. 00000100:00000040:0.0:1775776812.381332:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776812.381333:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776812.381334:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776812.381335:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775776812.381338:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00000001:3.0:1775776812.381339:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776812.381343:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776812.381345:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776812.381346:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1775776812.381347:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1775776812.381349:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776812.381349:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776812.381354:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381354:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776812.381357:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776812.381358:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776812.381360:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1775776812.381361:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000040:3.0:1775776812.381376:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1775776812.381386:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381386:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776812.381388:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.381389:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.381391:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776812.381393:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1775776812.381394:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 02000000:00000001:3.0:1775776812.381397:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776812.381398:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00100000:3.0:1775776812.381403:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797760:192.168.201.139@tcp:38:kworker.0 02000000:00000001:0.0:1775776812.381404:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776812.381406:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381409:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776812.381410:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776812.381411:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776812.381412:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776812.381413:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776812.381415:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.381416:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776812.381420:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115bc00. 02000000:00000010:0.0:1775776812.381422:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:3.0:1775776812.381424:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776812.381426:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776812.381430:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00100000:0.0:1775776812.381434:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1775776812.381434:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776812.381439:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776812.381444:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099797760, portal 10 00000100:00000001:0.0:1775776812.381447:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776812.381447:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776812.381450:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741753472 : -114841967798144 : ffff978d454dd880) 00000100:00000001:0.0:1775776812.381451:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776812.381456:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776812.381459:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381480:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776812.381482:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776812.381485:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776812.381487:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000040:0.0:1775776812.381489:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18537) 00000100:00000200:3.0:1775776812.381490:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099797760, offset 0 00000400:00000200:3.0:1775776812.381496:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776812.381501:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.381504:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776812.381507:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776812.381510:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776812.381513:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776812.381515:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776812.381518:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4463:4463:256:4294967295] 192.168.201.139@tcp LPNI seq info [4463:4463:8:4294967295] 00000400:00000200:3.0:1775776812.381522:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.381528:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776812.381532:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776812.381535:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776812.381539:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776812.381542:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775776812.381545:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776812.381548:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:3.0:1775776812.381551:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776812.381553:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776812.381555:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776812.381559:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776812.381561:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776812.381563:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776812.381565:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776812.381568:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381582:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.381584:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381591:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381592:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776812.381595:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776812.381598:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776812.381601:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776812.381603:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313200133 00000100:00000001:3.0:1775776812.381605:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313201398 00000100:00000001:3.0:1775776812.381607:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313200133 00000100:00000001:3.0:1775776812.381609:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313201398 00000100:00000001:3.0:1775776812.381611:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776812.381612:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381614:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776812.381617:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776812/real 1775776812] req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776812.381624:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776812.381625:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776812.381627:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776812.381629:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776812.381630:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776812.381632:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776812.381635:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776812.381643:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381649:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776812.381650:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381652:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.381653:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776812.381656:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381662:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776812.381666:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381673:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776812.381675:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381677:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.381678:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381679:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776812.381681:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776812.381683:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776812.381686:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776812.381688:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776812.381691:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776812.381698:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797760:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776812.381702:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776812.381705:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381711:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776812.381712:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.381714:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776812.381716:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776812.381717:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776812.381720:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797888:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776812.381723:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776812.381724:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776812.381726:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.381728:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776812.381730:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776812.381732:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776812.381735:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776812.381737:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776812.381740:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776812.381742:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099797888, portal 10 00000100:00000001:3.0:1775776812.381744:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776812.381745:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748992 : -114841967802624 : ffff978d454dc700) 00000100:00000040:3.0:1775776812.381750:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381756:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776812.381758:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776812.381760:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776812.381761:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099797888, offset 0 00000400:00000200:3.0:1775776812.381765:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776812.381768:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.381771:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776812.381774:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776812.381776:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776812.381794:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776812.381797:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776812.381800:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4464:4464:256:4294967295] 192.168.201.139@tcp LPNI seq info [4464:4464:8:4294967295] 00000400:00000200:3.0:1775776812.381804:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.381810:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776812.381814:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776812.381817:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776812.381820:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776812.381823:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775776812.381827:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776812.381829:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:3.0:1775776812.381833:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776812.381834:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776812.381837:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776812.381840:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776812.381843:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776812.381844:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776812.381846:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776812.381849:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381857:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.381859:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381866:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381868:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776812.381870:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776812.381872:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776812.381875:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776812.381876:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313200135 00000100:00000001:3.0:1775776812.381879:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313201398 00000100:00000001:3.0:1775776812.381880:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313200135 00000100:00000001:3.0:1775776812.381882:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313201398 00000100:00000001:3.0:1775776812.381884:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776812.381886:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.381888:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776812.381891:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776812/real 1775776812] req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776812.381898:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776812.381899:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776812.381901:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776812.381903:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776812.381904:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776812.381906:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776812.381909:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776812.381916:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381922:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776812.381923:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381925:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.381926:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776812.381930:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381936:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776812.381939:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381946:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776812.381948:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381949:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.381951:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.381952:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776812.381954:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776812.381956:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776812.381958:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776812.381960:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776812.381964:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776812.381976:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099797888:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776812.381980:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776812.381982:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.381988:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776812.381989:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.381990:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776812.381992:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776812.381993:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776812.381996:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798016:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776812.381998:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776812.382000:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776812.382001:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.382002:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776812.382005:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776812.382006:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776812.382009:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776812.382011:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776812.382013:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776812.382015:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099798016, portal 4 00000100:00000001:3.0:1775776812.382017:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776812.382018:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741750784 : -114841967800832 : ffff978d454dce00) 00000100:00000040:3.0:1775776812.382023:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382028:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776812.382030:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776812.382031:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776812.382033:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099798016, offset 0 00000400:00000200:3.0:1775776812.382036:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776812.382039:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.382041:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776812.382044:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776812.382046:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776812.382048:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776812.382050:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776812.382052:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4465:4465:256:4294967295] 192.168.201.139@tcp LPNI seq info [4465:4465:8:4294967295] 00000400:00000200:3.0:1775776812.382056:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.382061:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776812.382064:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776812.382067:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776812.382070:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776812.382072:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775776812.382075:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776812.382077:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:3.0:1775776812.382080:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776812.382082:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776812.382084:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776812.382087:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776812.382089:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776812.382090:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776812.382092:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776812.382094:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382100:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.382103:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382108:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.382110:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776812.382111:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776812.382114:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776812.382116:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776812.382118:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313200144 00000100:00000001:3.0:1775776812.382120:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313201398 00000100:00000001:3.0:1775776812.382122:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313200144 00000100:00000001:3.0:1775776812.382123:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313201398 00000100:00000001:3.0:1775776812.382125:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776812.382126:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.382128:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776812.382130:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776812/real 1775776812] req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776812.382136:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776812.382137:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776812.382139:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776812.382140:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776812.382142:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776812.382143:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776812.382146:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776812.382152:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382158:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776812.382159:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382160:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.382161:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776812.382164:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382169:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776812.382172:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382177:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776812.382179:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382180:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.382181:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382183:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776812.382184:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776812.382186:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776812.382189:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776812.382190:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776812.382193:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776812.382199:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798016:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776812.382202:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382203:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.382206:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd880 x1862033099797760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382211:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776812.382212:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776812.382213:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776812.382215:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776812.382216:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776812.382217:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776812.382219:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776812.382220:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:3.0:1775776812.382223:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776812.382224:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776812.382225:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd880. 00000100:00000001:3.0:1775776812.382227:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776812.382228:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382229:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.382232:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc700 x1862033099797888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382242:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776812.382243:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776812.382244:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776812.382246:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776812.382246:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776812.382247:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776812.382249:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776812.382250:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:3.0:1775776812.382251:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776812.382252:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776812.382253:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc700. 00000100:00000001:3.0:1775776812.382255:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776812.382256:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382257:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.382259:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dce00 x1862033099798016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382265:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776812.382266:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776812.382267:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776812.382269:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776812.382270:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776812.382271:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776812.382273:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776812.382274:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c000. 02000000:00000001:3.0:1775776812.382276:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776812.382277:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776812.382278:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dce00. 00000100:00000001:3.0:1775776812.382280:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776812.382281:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382283:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382286:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776812.382287:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.382289:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776812.382290:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776812.382292:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776812.382295:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382301:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776812.382302:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.382304:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776812.382306:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776812.382308:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776812.382310:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798144:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776812.382313:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776812.382314:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776812.382316:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776812.382318:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776812.382320:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776812.382322:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776812.382325:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776812.382327:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776812.382329:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776812.382332:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099798144, portal 4 00000100:00000001:3.0:1775776812.382334:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776812.382335:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748096 : -114841967803520 : ffff978d454dc380) 00000100:00000040:3.0:1775776812.382340:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382346:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776812.382348:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776812.382350:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776812.382352:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099798144, offset 0 00000400:00000200:3.0:1775776812.382356:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776812.382359:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.382372:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776812.382375:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776812.382378:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776812.382380:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776812.382382:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776812.382385:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4466:4466:256:4294967295] 192.168.201.139@tcp LPNI seq info [4466:4466:8:4294967295] 00000400:00000200:3.0:1775776812.382388:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776812.382394:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776812.382398:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776812.382401:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776812.382405:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776812.382407:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1775776812.382411:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776812.382413:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:3.0:1775776812.382417:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776812.382419:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776812.382421:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776812.382424:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776812.382427:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776812.382429:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776812.382431:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776812.382434:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382441:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.382444:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776837 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382450:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.382452:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776812.382454:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776812.382457:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776812.382460:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776812.382462:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313200135 00000100:00000001:3.0:1775776812.382464:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313201398 00000100:00000001:3.0:1775776812.382466:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313200135 00000100:00000001:3.0:1775776812.382468:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313201398 00000100:00000001:3.0:1775776812.382470:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776812.382472:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.382474:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776812.382477:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776812/real 1775776812] req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776812.382484:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776812.382486:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776812.382487:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776812.382489:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776812.382490:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776812.382492:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776812.382495:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776812.382504:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382509:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776812.382511:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382512:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.382513:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776812.382516:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382523:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776812.382526:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382533:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776812.382536:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382537:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776812.382538:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382540:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776812.382542:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776812.382544:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776812.382547:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776812.382549:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776812.382552:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776812.382559:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798144:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776812.382562:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382564:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776812.382566:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc380 x1862033099798144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776837 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776812.382573:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776812.382574:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776812.382576:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776812.382578:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776812.382579:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776812.382580:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776812.382582:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776812.382583:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759ec00. 02000000:00000001:3.0:1775776812.382585:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776812.382587:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776812.382588:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc380. 00000100:00000001:3.0:1775776812.382590:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776812.382591:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382593:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776812.382596:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776812.382597:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776812.382598:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776812.382600:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776813.083891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776813.083899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776813.083901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776813.083905:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776813.083909:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776813.083913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776813.083915:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776813.083917:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776813.083919:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776813.147907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776813.147914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776813.147920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776813.147932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776813.147937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776813.147938:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776813.147941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776813.147943:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776813.148064:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776813.148067:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776813.148069:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776813.211885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776813.211890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776813.211891:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776813.211895:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776813.211898:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776813.211901:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776813.211903:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776813.211905:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776813.211906:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776813.659889:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313201398, at 4313201408 00000100:00000010:0.0:1775776813.659903:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344360. 00000100:00000040:1.0:1775776813.659934:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313201398, at 4313201408 00000100:00000010:1.0:1775776813.659942:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503441e0. 00000100:00000001:2.0:1775776814.107903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776814.107910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776814.107912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776814.107917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776814.107920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776814.107924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776814.107926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776814.107929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776814.107931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776814.171894:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776814.171901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776814.171906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776814.171907:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776814.171910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776814.171912:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776814.171915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776814.171916:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776814.171985:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776814.171988:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776814.171991:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776814.235902:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776814.235909:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776814.235911:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776814.235917:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776814.235920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776814.235923:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776814.235925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776814.235928:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776814.235930:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776815.132214:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776815.132220:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776815.132222:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776815.132228:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776815.132232:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776815.132236:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776815.132239:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776815.132241:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776815.132244:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776815.195907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776815.195914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776815.195920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776815.195921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776815.195924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776815.195925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776815.195929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776815.195930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776815.196032:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776815.196034:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776815.196036:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776815.259905:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776815.259913:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776815.259915:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776815.259920:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776815.259923:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776815.259927:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776815.259929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776815.259932:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776815.259935:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776816.156018:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776816.156025:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776816.156027:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776816.156032:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776816.156035:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776816.156038:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776816.156040:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776816.156043:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776816.156045:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776816.219882:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776816.219889:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776816.219895:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776816.219897:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776816.219899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776816.219901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776816.219904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776816.219905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776816.221470:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776816.221474:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776816.221478:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776816.283982:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776816.283988:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776816.283990:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776816.283996:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776816.284000:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776816.284004:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776816.284007:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776816.284009:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776816.284012:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776817.179911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776817.179919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776817.179921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776817.179927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776817.179930:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776817.179934:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776817.179936:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776817.179938:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776817.179942:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776817.243898:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776817.243904:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776817.243910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776817.243912:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776817.243915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776817.243917:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776817.243920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776817.243921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776817.243986:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776817.243990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776817.243994:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776817.308117:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776817.308123:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776817.308125:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776817.308130:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776817.308134:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776817.308137:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776817.308140:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776817.308142:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776817.308145:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776817.499936:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776817.499947:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776817.499950:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776817.499954:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776817.499959:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776817.499966:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18532 00000400:00000200:0.0:1775776817.499974:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776817.499979:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776817.499982:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776817.499984:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.499986:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.499988:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.499990:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776817.499993:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776817.499994:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776817.499996:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776817.499998:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776817.499999:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776817.500000:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776817.500002:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500003:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.500005:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500006:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776817.500007:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776817.500009:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500011:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776817.500013:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776817.500015:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776817.500016:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500018:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776817.500019:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776817.500027:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776817.500030:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776817.500033:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776817.500035:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500038:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776817.500040:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776817.500044:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776817.500052:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776817.500058:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776817.500067:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776817.500082:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776817.500085:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776817.500088:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776817.500109:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776817.500112:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776817.500114:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776817.500115:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776817.500116:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776817.500118:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776817.500119:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18532 00000400:00000200:0.0:1775776817.500122:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776817.500124:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776817.500133:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776817.500136:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500138:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.500139:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500141:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776817.500143:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776817.500144:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776817.500146:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776817.500147:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776817.500149:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776817.500151:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776817.500154:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500156:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.500157:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500159:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776817.500162:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776817.500165:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500167:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776817.500169:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776817.500174:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776817.500176:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500178:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776817.500181:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776817.500183:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500184:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776817.500187:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776817.500188:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776817.500190:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776817.500191:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500192:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776817.500193:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776817.500194:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776817.500196:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776817.500198:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776817.500201:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776817.500206:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776817.500208:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776817.500210:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776817.500214:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776817.500215:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776817.500217:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776817.500218:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776817.500219:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776817.500221:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776817.500222:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18532 00000400:00000200:0.0:1775776817.500225:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776817.500227:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776817.500229:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776817.500230:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500232:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.500233:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500234:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776817.500235:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776817.500236:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776817.500237:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776817.500238:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776817.500239:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776817.500240:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776817.500242:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500243:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.500244:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500245:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776817.500246:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776817.500257:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500260:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776817.500261:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776817.500263:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776817.500264:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500265:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776817.500266:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776817.500267:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500268:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776817.500270:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776817.500271:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776817.500291:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776817.500293:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500294:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776817.500295:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776817.500296:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776817.500299:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776817.500300:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776817.500303:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776817.500308:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776817.500309:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776817.500311:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776817.500315:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776817.500316:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776817.500318:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776817.500319:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776817.500320:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776817.500321:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776817.500323:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18532 00000400:00000200:0.0:1775776817.500325:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776817.500327:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776817.500328:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776817.500329:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500331:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.500332:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500333:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776817.500335:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776817.500335:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776817.500337:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776817.500338:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776817.500340:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776817.500342:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776817.500345:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500347:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.500348:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500350:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776817.500352:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776817.500354:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776817.500356:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776817.500357:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776817.500360:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776817.500362:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500363:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776817.500365:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776817.500368:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500369:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776817.500372:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776817.500375:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776817.500377:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776817.500378:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.500379:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776817.500380:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776817.500382:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776817.500386:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776817.500389:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776817.500393:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776817.501631:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776817.501636:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776817.501639:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776817.501646:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.501656:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776817.501658:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.501662:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776817.501664:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776817.501666:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776817.501668:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099798272:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776817.501671:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776817.501674:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776817.501676:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.501677:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776817.501682:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776817.501683:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776817.501687:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776817.501690:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776817.501694:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776817.501699:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099798272, portal 25 00000100:00000001:3.0:1775776817.501701:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776817.501702:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000100:00000040:3.0:1775776817.501706:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.501711:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776817.501712:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776817.501714:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776817.501716:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099798272, offset 0 00000400:00000200:3.0:1775776817.501720:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776817.501726:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.501730:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776817.501733:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776817.501736:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776817.502234:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776817.502236:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776817.502239:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4467:4467:256:4294967295] 192.168.201.139@tcp LPNI seq info [4467:4467:8:4294967295] 00000400:00000200:3.0:1775776817.502242:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.502250:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776817.502254:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.502258:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776817.502263:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776817.502266:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775776817.502270:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776817.502272:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000100:3.0:1775776817.502292:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.502294:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776817.502301:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776817.502305:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776817.502314:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776817.502316:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776817.502320:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776817.502324:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.502333:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.502336:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.502343:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.502345:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776817.502350:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776817.502353:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776817.502356:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313206518 00000100:00000010:3.0:1775776817.502360:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503442a0. 00000100:00000001:3.0:1775776817.502366:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776817.502372:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313206518 00000100:00000010:3.0:1775776817.502375:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503447e0. 00000100:00000001:3.0:1775776817.502378:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776817.502381:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776817.502384:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.502386:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776817.502390:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776817/real 1775776817] req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776817.502406:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776817.502407:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776817.502409:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776817.502411:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776817.502413:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776817.502415:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776817.502418:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776817.502426:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.502433:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776817.502435:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.502437:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.502439:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776817.502442:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.502450:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776817.502454:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.502460:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776817.502463:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.502464:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.502465:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.502467:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776817.502470:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776817.502473:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776817.502681:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776817.502684:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776817.502688:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776817.502696:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099798272:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776817.502701:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776817.502704:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.502710:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776817.502711:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.502713:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776817.502716:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776817.502718:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776817.502720:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798400:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776817.502723:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776817.502725:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776817.502727:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.502728:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776817.502732:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776817.502733:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776817.502737:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776817.502811:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776817.502884:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776817.502888:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099798400, portal 10 00000100:00000001:3.0:1775776817.502952:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776817.502953:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:3.0:1775776817.502960:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.502967:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776817.502969:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776817.502972:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776817.502975:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099798400, offset 0 00000400:00000200:3.0:1775776817.502979:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776817.502988:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.502992:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776817.502995:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776817.502998:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776817.503000:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776817.503002:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776817.503005:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4468:4468:256:4294967295] 192.168.201.139@tcp LPNI seq info [4468:4468:8:4294967295] 00000400:00000200:3.0:1775776817.503008:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.503014:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776817.503018:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776817.503019:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:3.0:1775776817.503021:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000100:00000001:0.0:1775776817.503022:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1775776817.503025:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776817.503027:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775776817.503031:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776817.503034:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000100:00000040:0.0:1775776817.503037:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1775776817.503037:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.503039:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776817.503042:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776817.503046:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775776817.503047:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776817.503048:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776817.503050:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00080000:0.0:1775776817.503051:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775776817.503052:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1775776817.503055:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776817.503056:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:3.0:1775776817.503056:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776817.503058:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776817.503061:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776817.503064:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18532 00000100:00000001:3.0:1775776817.503064:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.503066:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776817.503068:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776817.503071:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:3.0:1775776817.503073:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776817.503074:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775776817.503074:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1775776817.503077:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776817.503077:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.503080:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776817.503080:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000100:00000040:0.0:1775776817.503082:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:3.0:1775776817.503083:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776817.503084:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:3.0:1775776817.503085:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313205255 00000020:00000001:0.0:1775776817.503087:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:3.0:1775776817.503088:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313206518 00000020:00000040:0.0:1775776817.503089:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000100:00000001:3.0:1775776817.503090:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313205255 00000020:00000001:0.0:1775776817.503091:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776817.503092:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313206518 00000020:00000040:0.0:1775776817.503093:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000100:00000001:3.0:1775776817.503094:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775776817.503096:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:3.0:1775776817.503096:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776817.503098:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:3.0:1775776817.503098:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1775776817.503100:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00100000:3.0:1775776817.503101:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776817/real 1775776817] req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776817.503102:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776817.503104:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776817.503106:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776817.503109:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000400:00000001:3.0:1775776817.503109:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776817.503110:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1775776817.503112:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000010:3.0:1775776817.503112:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000100:00000001:0.0:1775776817.503114:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776817.503114:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 02000000:00000001:0.0:1775776817.503116:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:3.0:1775776817.503116:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776817.503118:0:301888:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1775776817.503120:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1775776817.503120:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776817.503122:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.503124:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776817.503134:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776817.503137:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.503139:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776817.503143:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776817.503146:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776817.503149:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776817.503151:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776817.503154:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776817.503156:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776817.503159:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776817.503163:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776817.503168:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776817.503173:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776817.503180:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776817.503182:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776817.503185:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776817.503192:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1775776817.503192:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776817.503197:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18542) 00000100:00000001:3.0:1775776817.503199:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776817.503200:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503202:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.503203:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776817.503208:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503214:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776817.503217:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503224:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776817.503227:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503228:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.503229:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503231:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776817.503233:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776817.503235:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776817.503239:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776817.503240:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776817.503243:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776817.503251:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798400:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776817.503254:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776817.503258:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503264:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776817.503266:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.503268:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776817.503270:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776817.503272:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776817.503284:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798528:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776817.503288:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776817.503290:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776817.503291:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.503293:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776817.503296:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776817.503297:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776817.503301:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776817.503303:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776817.503306:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776817.503308:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099798528, portal 10 00000100:00000001:3.0:1775776817.503310:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776817.503312:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:3.0:1775776817.503317:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503324:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776817.503326:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776817.503328:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776817.503330:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099798528, offset 0 00000400:00000200:3.0:1775776817.503334:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776817.503338:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.503341:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776817.503344:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776817.503346:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776817.503349:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776817.503351:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776817.503354:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4469:4469:256:4294967295] 192.168.201.139@tcp LPNI seq info [4469:4469:8:4294967295] 00000400:00000200:3.0:1775776817.503357:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.503363:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776817.503367:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.503370:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776817.503374:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776817.503376:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775776817.503380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776817.503382:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000100:3.0:1775776817.503386:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.503387:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776817.503390:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776817.503393:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776817.503395:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776817.503397:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776817.503399:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776817.503403:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503417:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.503419:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503425:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.503427:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776817.503429:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776817.503432:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776817.503435:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776817.503437:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313205256 00000100:00000001:3.0:1775776817.503439:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313206518 00000100:00000001:3.0:1775776817.503441:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313205256 00000100:00000001:3.0:1775776817.503443:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313206518 00000100:00000001:3.0:1775776817.503445:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776817.503447:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.503449:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776817.503452:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776817/real 1775776817] req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776817.503460:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776817.503461:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776817.503463:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776817.503465:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776817.503466:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776817.503468:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776817.503471:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776817.503541:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503548:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776817.503550:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503552:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.503553:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776817.503556:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503562:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776817.503566:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503571:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776817.503574:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503575:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.503577:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503578:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776817.503580:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776817.503582:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776817.503585:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776817.503587:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776817.503591:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776817.503599:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798528:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776817.503602:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503604:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.503672:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099798272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503680:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776817.503683:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776817.503685:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776817.503688:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776817.503690:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776817.503692:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776817.503694:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776817.503696:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f800. 02000000:00000001:3.0:1775776817.503699:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776817.503701:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776817.503702:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:3.0:1775776817.503704:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776817.503705:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503706:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.503708:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099798400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503712:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776817.503713:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776817.503714:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776817.503715:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776817.503716:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776817.503716:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776817.503717:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776817.503718:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:3.0:1775776817.503719:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776817.503720:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776817.503720:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000100:00000001:3.0:1775776817.503721:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776817.503722:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503723:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.503724:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099798528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503727:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776817.503728:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776817.503729:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776817.503730:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776817.503730:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776817.503731:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776817.503732:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776817.503733:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f400. 02000000:00000001:3.0:1775776817.503734:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776817.503735:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776817.503735:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:3.0:1775776817.503736:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776817.503737:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503739:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.503806:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776817.503807:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.503809:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776817.503810:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776817.503811:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776817.503814:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.503958:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776817.503959:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.503961:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776817.503963:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776817.503965:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776817.503969:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798656:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776817.503972:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776817.503974:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776817.503975:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.503977:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776817.503980:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776817.503982:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776817.503986:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776817.503988:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776817.503992:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776817.503995:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099798656, portal 4 00000100:00000001:3.0:1775776817.503997:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776817.503998:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:3.0:1775776817.504003:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504010:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776817.504012:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776817.504014:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776817.504017:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099798656, offset 0 00000400:00000200:3.0:1775776817.504021:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776817.504025:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.504028:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776817.504031:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776817.504033:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776817.504036:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776817.504038:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776817.504040:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4470:4470:256:4294967295] 192.168.201.139@tcp LPNI seq info [4470:4470:8:4294967295] 00000400:00000200:3.0:1775776817.504044:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.504049:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776817.504053:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.504056:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776817.504060:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776817.504062:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775776817.504067:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776817.504069:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000100:3.0:1775776817.504086:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.504088:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776817.504091:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776817.504094:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776817.504097:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776817.504099:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776817.504101:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776817.504104:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504109:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.504111:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504115:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.504116:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776817.504118:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776817.504120:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776817.504122:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776817.504123:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313205266 00000100:00000001:3.0:1775776817.504364:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313206518 00000100:00000001:3.0:1775776817.504367:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313205266 00000100:00000001:3.0:1775776817.504369:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313206518 00000100:00000001:3.0:1775776817.504371:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776817.504374:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.504376:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776817.504380:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776817/real 1775776817] req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776817.504388:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776817.504389:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776817.504391:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776817.504393:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776817.504395:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776817.504397:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776817.504399:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776817.504407:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504414:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776817.504415:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.504417:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.504418:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776817.504421:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504427:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776817.504431:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504437:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776817.504439:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.504441:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.504442:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.504444:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776817.504445:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776817.504448:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776817.504451:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776817.504454:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776817.504457:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776817.504465:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798656:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776817.504468:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776817.504471:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504547:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776817.504549:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.504551:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776817.504553:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776817.504554:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776817.504557:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798784:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776817.504560:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776817.504562:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776817.504563:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776817.504565:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776817.504568:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776817.504570:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776817.504573:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776817.504576:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776817.504579:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776817.504582:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099798784, portal 4 00000100:00000001:3.0:1775776817.504584:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776817.504585:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:3.0:1775776817.504590:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504596:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776817.504598:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776817.504600:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776817.504603:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099798784, offset 0 00000400:00000200:3.0:1775776817.504675:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776817.504679:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.504682:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776817.504684:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776817.504686:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776817.504688:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776817.504689:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776817.504691:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4471:4471:256:4294967295] 192.168.201.139@tcp LPNI seq info [4471:4471:8:4294967295] 00000400:00000200:3.0:1775776817.504693:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776817.504698:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776817.504700:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.504703:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776817.504705:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776817.504707:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1775776817.504709:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776817.504711:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 23 secs later) 00000800:00000100:3.0:1775776817.504713:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776817.504714:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776817.504717:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776817.504719:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776817.504721:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776817.504722:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776817.504723:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776817.504726:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504731:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.504733:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504736:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.504737:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776817.504885:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776817.504888:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776817.504946:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776817.504948:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313205258 00000100:00000001:3.0:1775776817.504950:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313206518 00000100:00000001:3.0:1775776817.504952:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313205258 00000100:00000001:3.0:1775776817.504954:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313206518 00000100:00000001:3.0:1775776817.504956:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776817.504957:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.504959:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776817.504963:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776817/real 1775776817] req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776817.504970:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776817.504971:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776817.504972:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776817.504974:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776817.504976:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776817.504978:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776817.504980:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776817.504988:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.504999:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776817.505000:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.505001:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.505003:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776817.505005:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.505011:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776817.505014:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.505020:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776817.505022:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.505023:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776817.505025:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.505027:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776817.505029:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776817.505031:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776817.505034:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776817.505036:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776817.505040:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776817.505046:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099798784:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776817.505050:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.505052:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.505054:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099798656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.505060:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776817.505062:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776817.505065:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776817.505067:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776817.505069:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776817.505070:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776817.505072:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776817.505074:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:3.0:1775776817.505077:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776817.505078:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776817.505080:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:3.0:1775776817.505082:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776817.505083:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.505085:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776817.505088:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099798784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776817.505093:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776817.505094:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776817.505095:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776817.505097:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776817.505098:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776817.505099:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776817.505101:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776817.505102:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e400. 02000000:00000001:3.0:1775776817.505104:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776817.505105:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776817.505106:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:3.0:1775776817.505108:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776817.505109:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.505111:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776817.505115:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776817.505116:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776817.505118:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776817.505119:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776818.203890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776818.203897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776818.203898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776818.203904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776818.203908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776818.203911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776818.203914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776818.203915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776818.203918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776818.267897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776818.267905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776818.267911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776818.267913:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776818.267916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776818.267918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776818.267921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776818.267928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776818.268059:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776818.268063:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776818.268066:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776818.332544:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776818.332551:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776818.332553:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776818.332558:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776818.332562:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776818.332567:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776818.332569:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776818.332572:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776818.332575:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776818.779909:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313206518, at 4313206528 00000100:00000010:0.0:1775776818.779924:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503442a0. 00000100:00000040:1.0:1775776818.780133:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313206518, at 4313206528 00000100:00000010:1.0:1775776818.780140:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503447e0. 00000100:00000001:2.0:1775776819.227904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776819.227911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776819.227912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776819.227917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776819.227921:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776819.227924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776819.227927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776819.227929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776819.227932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776819.291964:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776819.291983:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776819.291986:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776819.291990:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776819.291994:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776819.291996:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776819.291998:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776819.292000:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776819.292004:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776819.292006:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776819.292009:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776819.292010:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776819.292013:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776819.292014:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776819.292136:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776819.292140:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776819.292142:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776819.356153:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776819.356158:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776819.356160:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776819.356165:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776819.356168:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776819.356171:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776819.356174:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776819.356176:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776819.356179:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776820.252326:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776820.252334:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776820.252335:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776820.252341:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776820.252344:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776820.252347:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776820.252350:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776820.252352:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776820.252355:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776820.315901:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776820.315908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776820.315914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776820.315915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776820.315918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776820.315919:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776820.315923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776820.315924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776820.316048:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776820.316051:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776820.316054:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776820.380265:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776820.380270:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776820.380272:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776820.380283:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776820.380286:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776820.380290:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776820.380292:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776820.380294:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776820.380296:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776821.275859:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776821.275866:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776821.275868:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776821.275873:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776821.275876:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776821.275880:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776821.275882:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776821.275884:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776821.275887:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776821.339947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339952:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776821.339956:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776821.339958:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776821.339962:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339965:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776821.339967:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339969:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339971:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339974:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776821.339975:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776821.339976:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776821.339977:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339979:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776821.339980:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339981:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339983:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339986:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776821.339987:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776821.339988:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776821.339989:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339991:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776821.339992:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339993:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776821.339995:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776821.340127:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776821.340131:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776821.340134:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776821.403909:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776821.403916:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776821.403918:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776821.403924:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776821.403927:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776821.403931:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776821.403933:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776821.404109:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776821.404113:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.299905:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776822.299914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776822.299916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776822.299921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776822.299924:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776822.299928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776822.299931:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776822.299934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776822.299936:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776822.363889:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776822.363893:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776822.363897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776822.363898:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776822.363901:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776822.363902:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776822.363904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776822.363904:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776822.364236:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776822.364240:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776822.364244:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776822.427860:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776822.427867:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776822.427869:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776822.427874:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776822.427878:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776822.427881:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776822.427883:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776822.427885:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776822.427888:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776822.619942:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776822.619951:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776822.619954:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776822.619957:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776822.619960:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776822.619967:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18537 00000400:00000200:2.0:1775776822.619976:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776822.619981:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776822.619984:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776822.619988:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.619991:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.619994:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.619996:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776822.620000:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776822.620002:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776822.620004:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776822.620007:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776822.620009:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776822.620012:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776822.620014:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.620016:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.620018:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.620020:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776822.620023:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776822.620026:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776822.620028:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776822.620030:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776822.620034:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776822.620037:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.620039:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776822.620041:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776822.620049:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ea00. 00000020:00000040:2.0:1775776822.620052:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776822.620056:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776822.620058:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.620060:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776822.620062:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776822.620066:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776822.620073:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:2.0:1775776822.620080:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776822.620088:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776822.620098:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776822.620102:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776822.620106:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc493d27] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776822.620131:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776822.620134:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776822.620137:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776822.620139:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776822.620140:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776822.620143:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776822.620145:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18537 00000400:00000200:2.0:1775776822.620149:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776822.620151:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776822.620154:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776822.620156:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.620158:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.620160:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.620162:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776822.620165:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776822.620166:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776822.620167:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775776822.620372:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776822.620375:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776822.620378:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776822.620384:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.620398:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776822.620399:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.620402:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776822.620404:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776822.620406:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776822.620410:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099798912:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776822.620414:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776822.620426:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776822.620429:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.620431:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776822.620436:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776822.620438:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776822.620443:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776822.620447:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776822.620454:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776822.620461:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099798912, portal 25 00000100:00000001:3.0:1775776822.620463:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776822.620465:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288721408 : -114841420830208 : ffff978d65e7ea00) 00000100:00000040:3.0:1775776822.620470:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.620477:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776822.620480:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776822.620482:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776822.620485:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099798912, offset 0 00000400:00000200:3.0:1775776822.620490:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776822.620498:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.620503:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776822.620508:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776822.620511:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776822.620515:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776822.620518:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776822.620522:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4472:4472:256:4294967295] 192.168.201.139@tcp LPNI seq info [4472:4472:8:4294967295] 00000400:00000200:3.0:1775776822.620526:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.620537:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776822.620542:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.620547:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776822.620553:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776822.620557:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776822.620562:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776822.620565:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776822.620569:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.620572:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776822.620577:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776822.620581:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776822.620586:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776822.620588:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776822.620592:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776822.620595:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.620604:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.620607:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.620613:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.620615:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776822.620619:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776822.620623:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776822.620626:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313211637 00000100:00000010:3.0:1775776822.620630:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503445a0. 00000100:00000001:3.0:1775776822.620636:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776822.620652:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313211637 00000100:00000010:3.0:1775776822.620655:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344ae0. 00000100:00000001:3.0:1775776822.620658:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776822.620661:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776822.620664:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.620666:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776822.620670:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776822/real 1775776822] req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776822.620680:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776822.620681:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776822.620683:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776822.620685:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776822.620686:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776822.620689:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776822.620692:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776822.620700:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.620707:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776822.620708:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.620711:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.620712:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776822.620715:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.620722:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776822.620726:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.620733:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776822.620736:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.620737:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.620738:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.620740:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776822.620742:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776822.620744:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776822.620748:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776822.620750:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776822.620754:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776822.620761:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099798912:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776822.620766:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.620768:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.620770:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ea00 x1862033099798912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.620776:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776822.620819:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776822.620822:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776822.620825:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776822.620826:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776822.620828:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776822.620830:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776822.620833:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:3.0:1775776822.620836:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776822.620838:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776822.620840:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ea00. 00000100:00000001:3.0:1775776822.620843:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776822.620844:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.620846:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.620852:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776822.620854:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.620857:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776822.620858:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776822.621126:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776822.621130:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776822.621133:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776822.621136:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621138:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.621140:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621143:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776822.621145:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776822.621148:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621150:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776822.621152:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776822.621157:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776822.621159:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621161:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776822.621164:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776822.621166:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621168:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776822.621172:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e680. 00000020:00000040:2.0:1775776822.621175:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776822.621177:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776822.621178:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621444:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776822.621446:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776822.621448:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776822.621453:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507000. 00000100:00000001:2.0:1775776822.621457:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776822.621462:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776822.621470:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776822.621472:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776822.621476:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086433f13] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776822.621491:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776822.621494:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776822.621498:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776822.621499:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776822.621501:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776822.621503:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776822.621506:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18537 00000400:00000200:2.0:1775776822.621510:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776822.621514:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776822.621516:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776822.621518:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621520:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.621522:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621524:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776822.621526:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776822.621527:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776822.621529:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776822.621530:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776822.621532:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776822.621535:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776822.621537:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621539:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.621540:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621542:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776822.621544:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776822.621547:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621549:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776822.621551:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776822.621554:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776822.621555:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621557:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776822.621559:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776822.621561:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621563:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776822.621566:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7fb80. 00000020:00000040:2.0:1775776822.621568:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776822.621571:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776822.621572:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621574:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776822.621575:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776822.621576:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776822.621579:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:2.0:1775776822.621581:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776822.621585:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776822.621592:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776822.621594:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776822.621597:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1787ebf] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776822.621604:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776822.621606:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776822.621609:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776822.621610:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776822.621612:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776822.621614:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776822.621616:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18537 00000400:00000200:2.0:1775776822.621619:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776822.621622:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776822.621624:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776822.621626:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621628:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.621630:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621632:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776822.621633:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776822.621634:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776822.621636:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776822.621637:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776822.621639:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776822.621641:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776822.621644:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621646:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.621647:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621649:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776822.621651:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776822.621654:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621656:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776822.621657:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776822.621659:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776822.621661:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621662:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776822.621664:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776822.621666:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621668:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776822.621671:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e300. 00000020:00000040:2.0:1775776822.621673:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776822.621675:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776822.621676:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621677:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776822.621679:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776822.621680:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776822.621683:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775776822.621686:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776822.621690:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776822.621696:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776822.621698:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776822.621701:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff8c8494] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776822.621709:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776822.621711:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776822.621714:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776822.621715:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776822.621716:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776822.621718:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776822.621720:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18537 00000400:00000200:2.0:1775776822.621723:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776822.621725:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776822.621727:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776822.621730:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621732:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.621734:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621735:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776822.621737:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776822.621738:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776822.621739:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776822.621740:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776822.621742:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776822.621745:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776822.621748:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621750:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776822.621751:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621753:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776822.621755:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776822.621757:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776822.621759:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776822.621760:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776822.621762:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776822.621764:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621766:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776822.621768:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776822.621769:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621771:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776822.621774:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c000. 00000020:00000040:2.0:1775776822.621775:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776822.621792:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776822.621794:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776822.621795:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776822.621796:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776822.621798:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776822.621801:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505000. 00000100:00000001:2.0:1775776822.621804:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776822.621808:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776822.621814:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776822.621816:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776822.621819:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4798067] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776822.621826:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776822.621830:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18548) 00000100:00000001:3.0:1775776822.622352:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776822.622355:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776822.622357:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776822.622362:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622370:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776822.622372:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.622375:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776822.622377:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776822.622379:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776822.622382:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799040:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776822.622385:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776822.622387:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776822.622389:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.622391:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776822.622395:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776822.622397:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776822.622401:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776822.622404:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776822.622408:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776822.622412:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099799040, portal 10 00000100:00000001:3.0:1775776822.622414:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776822.622426:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288720512 : -114841420831104 : ffff978d65e7e680) 00000100:00000040:3.0:1775776822.622432:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622439:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776822.622441:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776822.622443:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776822.622446:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099799040, offset 0 00000400:00000200:3.0:1775776822.622451:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776822.622456:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.622459:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776822.622463:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776822.622466:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776822.622469:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776822.622471:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776822.622474:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4473:4473:256:4294967295] 192.168.201.139@tcp LPNI seq info [4473:4473:8:4294967295] 00000400:00000200:3.0:1775776822.622478:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.622485:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776822.622491:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.622494:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776822.622499:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776822.622501:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776822.622506:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776822.622508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776822.622512:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.622514:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776822.622517:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776822.622521:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776822.622524:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776822.622525:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776822.622528:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776822.622531:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622539:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.622541:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622548:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.622550:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776822.622553:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776822.622557:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776822.622560:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776822.622562:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313210374 00000100:00000001:3.0:1775776822.622564:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313211637 00000100:00000001:3.0:1775776822.622566:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313210374 00000100:00000001:3.0:1775776822.622568:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313211637 00000100:00000001:3.0:1775776822.622570:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776822.622572:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.622574:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776822.622577:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776822/real 1775776822] req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776822.622585:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776822.622586:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776822.622588:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776822.622590:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776822.622591:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776822.622593:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776822.622596:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776822.622603:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622610:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776822.622611:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.622613:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.622614:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776822.622617:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622623:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776822.622627:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622633:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776822.622635:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.622637:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.622638:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.622640:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776822.622642:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776822.622645:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776822.622648:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776822.622650:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776822.622654:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776822.622662:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799040:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776822.622666:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776822.622668:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622675:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776822.622676:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.622678:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776822.622680:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776822.622681:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776822.622684:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799168:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776822.622687:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776822.622688:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776822.622690:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.622691:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776822.622694:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776822.622696:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776822.622699:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776822.622701:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776822.622704:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776822.622706:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099799168, portal 10 00000100:00000001:3.0:1775776822.622708:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776822.622709:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288725888 : -114841420825728 : ffff978d65e7fb80) 00000100:00000040:3.0:1775776822.622714:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622721:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776822.622723:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776822.622725:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776822.622727:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099799168, offset 0 00000400:00000200:3.0:1775776822.622730:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776822.622733:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.622736:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776822.622739:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776822.622741:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776822.622743:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776822.622745:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776822.622748:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4474:4474:256:4294967295] 192.168.201.139@tcp LPNI seq info [4474:4474:8:4294967295] 00000400:00000200:3.0:1775776822.622753:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.622758:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776822.622762:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.622765:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776822.622769:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776822.622771:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776822.622775:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776822.622793:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776822.622798:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.622799:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776822.622812:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776822.622815:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776822.622818:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776822.622819:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776822.622821:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776822.622825:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622832:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.622834:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622841:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.622842:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776822.622845:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776822.622847:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776822.622850:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776822.622852:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313210376 00000100:00000001:3.0:1775776822.622855:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313211637 00000100:00000001:3.0:1775776822.622856:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313210376 00000100:00000001:3.0:1775776822.622858:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313211637 00000100:00000001:3.0:1775776822.622860:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776822.622862:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.622864:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776822.622867:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776822/real 1775776822] req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776822.622874:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776822.622875:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776822.622877:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776822.622879:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776822.622880:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776822.622882:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776822.622885:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776822.622892:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622899:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776822.622900:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.622901:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.622903:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776822.622913:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622925:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776822.622929:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622935:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776822.622938:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.622940:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.622941:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.622943:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776822.622944:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776822.622946:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776822.622949:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776822.622952:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776822.622955:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776822.622963:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799168:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776822.622966:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776822.622969:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.622974:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776822.622975:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.622977:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776822.622987:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776822.622988:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776822.622991:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799296:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776822.622994:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776822.622996:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776822.622997:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.622999:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776822.623002:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776822.623003:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776822.623007:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776822.623009:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776822.623012:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776822.623014:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099799296, portal 4 00000100:00000001:3.0:1775776822.623016:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776822.623017:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288719616 : -114841420832000 : ffff978d65e7e300) 00000100:00000040:3.0:1775776822.623022:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.623028:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776822.623030:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776822.623032:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776822.623034:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099799296, offset 0 00000400:00000200:3.0:1775776822.623038:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776822.623041:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.623044:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776822.623047:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776822.623049:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776822.623051:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776822.623053:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776822.623056:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4475:4475:256:4294967295] 192.168.201.139@tcp LPNI seq info [4475:4475:8:4294967295] 00000400:00000200:3.0:1775776822.623060:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.623065:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776822.623069:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.623072:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776822.623075:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776822.623078:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776822.623081:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776822.623084:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776822.623088:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.623089:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776822.623092:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776822.623095:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776822.623098:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776822.623099:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776822.623101:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776822.623104:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.623117:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.623120:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.623126:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.623127:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776822.623130:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776822.623132:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776822.623135:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776822.623137:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313210385 00000100:00000001:3.0:1775776822.623139:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313211637 00000100:00000001:3.0:1775776822.623141:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313210385 00000100:00000001:3.0:1775776822.623143:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313211637 00000100:00000001:3.0:1775776822.623145:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776822.623147:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.623149:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776822.623152:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776822/real 1775776822] req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776822.623158:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776822.623159:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776822.623161:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776822.623162:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776822.623164:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776822.623166:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776822.623169:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776822.624517:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.624527:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776822.624530:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.624532:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.624533:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776822.624537:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.624544:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776822.624548:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.624555:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776822.624557:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.624558:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.624559:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.624561:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776822.624563:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776822.624565:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776822.624568:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776822.624570:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776822.624574:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776822.624581:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799296:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776822.624585:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776822.624588:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.624594:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776822.624595:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.624597:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776822.624599:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776822.624601:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776822.624604:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799424:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776822.624607:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776822.624609:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776822.624611:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776822.624612:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776822.624618:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776822.624620:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776822.624625:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776822.624628:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776822.624632:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776822.624636:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099799424, portal 4 00000100:00000001:3.0:1775776822.624637:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776822.624639:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288710656 : -114841420840960 : ffff978d65e7c000) 00000100:00000040:3.0:1775776822.624644:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.624651:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776822.624653:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776822.624655:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776822.624658:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099799424, offset 0 00000400:00000200:3.0:1775776822.624662:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776822.624666:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.624670:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776822.624673:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776822.624675:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776822.624677:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776822.624680:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776822.624683:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4476:4476:256:4294967295] 192.168.201.139@tcp LPNI seq info [4476:4476:8:4294967295] 00000400:00000200:3.0:1775776822.624687:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776822.624693:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776822.624698:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.624701:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776822.624705:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776822.624708:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775776822.624712:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776822.624715:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1775776822.624718:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776822.624720:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776822.624723:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776822.624727:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776822.624730:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776822.624731:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776822.624734:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776822.624736:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.624744:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.624747:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776847 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.624753:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.624754:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776822.624757:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776822.624759:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776822.624762:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776822.624764:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313210377 00000100:00000001:3.0:1775776822.624766:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313211637 00000100:00000001:3.0:1775776822.624768:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313210377 00000100:00000001:3.0:1775776822.624770:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313211637 00000100:00000001:3.0:1775776822.624772:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776822.624774:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.624776:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776822.624795:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776822/real 1775776822] req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776822.624869:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776822.624870:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776822.624872:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776822.624875:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776822.624876:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776822.624878:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776822.624882:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776822.624890:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.624896:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776822.624898:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.624899:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.624900:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776822.624903:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.625121:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776822.625126:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.625133:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776822.625135:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.625137:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776822.625138:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.625139:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776822.625141:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776822.625143:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776822.625146:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776822.625148:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776822.625152:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776822.625166:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799424:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776822.625169:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.625171:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.625174:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e680 x1862033099799040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.625181:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776822.625192:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776822.625195:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776822.625198:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776822.625200:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776822.625201:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776822.625203:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776822.625205:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507000. 02000000:00000001:3.0:1775776822.625208:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776822.625210:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776822.625212:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e680. 00000100:00000001:3.0:1775776822.625215:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776822.625216:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.625218:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.625221:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7fb80 x1862033099799168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.625332:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776822.625334:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776822.625335:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776822.625339:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776822.625340:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776822.625341:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776822.625343:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776822.625345:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:3.0:1775776822.625347:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776822.625348:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776822.625349:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7fb80. 00000100:00000001:3.0:1775776822.625351:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776822.625353:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.625354:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.625358:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e300 x1862033099799296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.625364:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776822.625365:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776822.625366:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776822.625368:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776822.625369:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776822.625370:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776822.625372:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776822.625373:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:3.0:1775776822.625375:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776822.625376:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776822.625377:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e300. 00000100:00000001:3.0:1775776822.625378:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776822.625380:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.625381:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776822.625383:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c000 x1862033099799424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776822.625388:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776822.625389:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776822.625391:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776822.625392:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776822.625394:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776822.625395:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776822.625396:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776822.625398:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:3.0:1775776822.625399:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776822.625400:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776822.625401:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c000. 00000100:00000001:3.0:1775776822.625403:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776822.625404:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.625406:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776822.625413:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776822.625415:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776822.625417:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776822.625477:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776823.323969:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776823.323976:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776823.323977:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776823.323983:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776823.323986:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776823.323990:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776823.323992:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776823.323994:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776823.323996:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776823.387894:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776823.387899:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776823.387904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776823.387905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776823.387908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776823.387909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776823.387912:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776823.387913:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776823.388025:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776823.388028:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776823.388031:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776823.451884:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776823.451890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776823.451892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776823.451898:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776823.451901:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776823.451905:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776823.451907:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776823.451909:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776823.451913:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776823.899931:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313211637, at 4313211648 00000100:00000010:1.0:1775776823.899944:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ae0. 00000100:00000040:0.0:1775776823.900217:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313211637, at 4313211648 00000100:00000010:0.0:1775776823.900224:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503445a0. 00000100:00000001:2.0:1775776824.347877:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776824.347890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776824.347892:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776824.347897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776824.347901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776824.347904:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776824.347907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776824.347909:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776824.347912:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776824.411903:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776824.411910:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776824.411916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776824.411917:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776824.411920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776824.411921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776824.411925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776824.411925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776824.411979:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776824.411982:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776824.411984:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776824.475853:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776824.475860:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776824.475862:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776824.475868:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776824.475871:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776824.475874:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776824.475877:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776824.475879:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776824.475882:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776825.371955:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776825.371962:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776825.371963:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776825.371970:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776825.371973:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776825.371977:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776825.371979:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776825.371981:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776825.371984:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776825.435905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776825.435911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776825.435917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776825.435918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776825.435921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776825.435923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776825.435926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776825.435927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776825.436018:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776825.436021:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776825.436025:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776825.501281:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776825.501288:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776825.501290:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776825.501296:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776825.501299:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776825.501303:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776825.501315:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776825.501318:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776825.501320:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776826.395966:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776826.395973:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776826.395975:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776826.395981:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776826.395984:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776826.395988:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776826.395990:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776826.395992:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776826.395995:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776826.461254:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776826.461260:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776826.461266:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776826.461267:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776826.461270:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776826.461271:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776826.461274:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776826.461276:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776826.461387:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776826.461390:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776826.461394:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776826.524128:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776826.524136:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776826.524138:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776826.524144:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776826.524148:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776826.524152:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776826.524154:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776826.524156:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776826.524159:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776827.419913:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776827.419921:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776827.419923:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776827.419928:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776827.419932:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776827.419935:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776827.419937:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776827.419940:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776827.419943:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776827.483942:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776827.483949:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776827.483956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776827.483957:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776827.483960:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776827.483962:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776827.483965:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776827.483966:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776827.484062:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776827.484065:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776827.484069:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776827.547920:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776827.547927:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776827.547929:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776827.547935:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776827.547938:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776827.547942:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776827.547945:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776827.547947:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776827.547950:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776827.739987:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776827.739997:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776827.740000:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776827.740004:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776827.740008:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776827.740016:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18543 00000400:00000200:0.0:1775776827.740025:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776827.740031:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776827.740034:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776827.740037:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740040:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740043:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740045:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776827.740049:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776827.740051:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776827.740053:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776827.740056:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776827.740058:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776827.740060:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776827.740074:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740076:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740077:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740079:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776827.740082:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776827.740085:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740087:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776827.740091:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776827.740103:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776827.740107:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740109:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776827.740113:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776827.740122:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775776827.740125:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776827.740129:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776827.740130:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740134:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776827.740136:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776827.740140:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776827.740147:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776827.740162:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776827.740171:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776827.740180:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776827.740184:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776827.740189:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776827.740212:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776827.740215:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776827.740218:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776827.740220:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776827.740221:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776827.740224:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776827.740226:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18543 00000400:00000200:0.0:1775776827.740230:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776827.740233:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776827.740235:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776827.740237:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740239:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740241:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740243:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776827.740245:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776827.740247:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776827.740249:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776827.740251:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776827.740252:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776827.740254:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776827.740257:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740259:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740260:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740262:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776827.740265:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776827.740267:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740269:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776827.740271:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776827.740276:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776827.740278:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740280:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776827.740284:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776827.740286:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740288:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776827.740291:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776827.740293:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776827.740296:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776827.740297:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740298:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776827.740300:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776827.740301:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776827.740305:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776827.740308:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776827.740312:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776827.740319:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776827.740321:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776827.740325:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776827.740332:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776827.740334:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776827.740337:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776827.740339:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776827.740340:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776827.740343:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776827.740345:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18543 00000400:00000200:0.0:1775776827.740348:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776827.740351:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776827.740353:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776827.740355:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740357:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740359:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740361:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776827.740363:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776827.740364:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776827.740366:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776827.740367:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776827.740369:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776827.740371:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776827.740373:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740375:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740377:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740379:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776827.740381:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776827.740383:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740386:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776827.740387:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776827.740390:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776827.740392:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740394:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776827.740396:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776827.740398:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740400:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776827.740404:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776827.740406:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776827.740408:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776827.740409:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740411:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776827.740412:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776827.740414:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776827.740417:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776827.740420:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776827.740423:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776827.740430:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776827.740432:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776827.740436:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776827.740443:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776827.740445:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776827.740448:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776827.740449:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776827.740451:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776827.740453:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776827.740455:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18543 00000400:00000200:0.0:1775776827.740459:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776827.740461:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776827.740463:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776827.740466:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740468:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740470:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740471:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776827.740473:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776827.740474:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776827.740476:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776827.740477:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776827.740479:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776827.740481:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776827.740484:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740485:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740487:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740489:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776827.740491:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776827.740493:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740495:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776827.740497:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776827.740500:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776827.740501:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740503:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776827.740505:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776827.740507:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740508:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776827.740512:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776827.740514:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776827.740516:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776827.740517:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740518:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776827.740520:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776827.740521:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776827.740524:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776827.740527:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776827.740532:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776827.740537:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776827.740539:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776827.740542:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776827.740548:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776827.740550:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776827.740553:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776827.740554:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776827.740555:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776827.740557:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776827.740560:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18543 00000400:00000200:0.0:1775776827.740573:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776827.740576:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776827.740578:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776827.740580:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740582:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740592:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740593:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776827.740595:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776827.740596:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776827.740597:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776827.740599:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776827.740600:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776827.740602:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776827.740604:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740605:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776827.740607:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740609:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776827.740610:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776827.740613:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776827.740615:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776827.740616:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776827.740625:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776827.740626:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740628:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776827.740629:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776827.740631:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740633:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776827.740636:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776827.740638:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776827.740639:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776827.740640:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776827.740642:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776827.740643:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776827.740644:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776827.740648:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776827.740651:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776827.740654:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776827.740660:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776827.740662:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776827.740665:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776827.740672:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776827.740676:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18553) 00000100:00000001:3.0:1775776827.740998:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776827.741002:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776827.741004:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776827.741010:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741017:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776827.741019:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.741022:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776827.741025:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776827.741027:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776827.741030:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099799552:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776827.741034:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776827.741037:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776827.741040:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.741042:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776827.741046:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776827.741048:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776827.741053:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776827.741057:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776827.741063:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776827.741081:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099799552, portal 25 00000100:00000001:3.0:1775776827.741084:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776827.741085:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741762432 : -114841967789184 : ffff978d454dfb80) 00000100:00000040:3.0:1775776827.741090:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741106:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776827.741109:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776827.741111:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776827.741116:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099799552, offset 0 00000400:00000200:3.0:1775776827.741121:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776827.741128:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.741134:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776827.741138:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776827.741141:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776827.741145:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776827.741147:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776827.741151:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4477:4477:256:4294967295] 192.168.201.139@tcp LPNI seq info [4477:4477:8:4294967295] 00000400:00000200:3.0:1775776827.741155:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.741165:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776827.741170:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.741175:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776827.741181:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776827.741184:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776827.741189:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776827.741192:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776827.741197:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.741199:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776827.741204:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776827.741208:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776827.741212:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776827.741214:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776827.741219:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776827.741223:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741231:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.741234:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741240:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.741242:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776827.741247:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776827.741250:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776827.741253:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313216758 00000100:00000010:3.0:1775776827.741257:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344600. 00000100:00000001:3.0:1775776827.741262:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776827.741268:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313216758 00000100:00000010:3.0:1775776827.741271:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344c00. 00000100:00000001:3.0:1775776827.741274:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776827.741277:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776827.741279:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.741281:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776827.741285:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776827/real 1775776827] req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776827.741293:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776827.741295:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776827.741297:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776827.741299:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776827.741300:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776827.741303:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776827.741306:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776827.741314:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741320:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776827.741322:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.741324:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.741326:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776827.741329:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741336:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776827.741339:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741346:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776827.741349:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.741350:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.741352:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.741354:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776827.741356:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776827.741358:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776827.741362:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776827.741364:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776827.741368:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776827.741375:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099799552:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776827.741380:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776827.741383:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741389:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776827.741390:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.741392:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776827.741393:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776827.741395:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776827.741398:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799680:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776827.741401:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776827.741402:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776827.741404:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.741405:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776827.741408:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776827.741410:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776827.741413:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776827.741416:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776827.741419:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776827.741421:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099799680, portal 10 00000100:00000001:3.0:1775776827.741423:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776827.741424:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757056 : -114841967794560 : ffff978d454de680) 00000100:00000040:3.0:1775776827.741431:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741437:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776827.741439:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776827.741441:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776827.741443:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099799680, offset 0 00000400:00000200:3.0:1775776827.741446:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776827.741450:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.741453:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776827.741455:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776827.741457:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776827.741460:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776827.741462:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776827.741465:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4478:4478:256:4294967295] 192.168.201.139@tcp LPNI seq info [4478:4478:8:4294967295] 00000400:00000200:3.0:1775776827.741469:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.741474:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776827.741478:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.741481:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776827.741485:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776827.741487:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776827.741491:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776827.741493:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776827.741497:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.741499:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776827.741501:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776827.741504:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776827.741507:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776827.741509:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776827.741511:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776827.741514:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741521:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.741523:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.741529:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.741531:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776827.741533:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776827.741537:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775776827.741540:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776827.741542:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313215493 00000100:00000001:3.0:1775776827.741544:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313216758 00000100:00000001:3.0:1775776827.741546:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313215493 00000100:00000001:3.0:1775776827.741548:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313216758 00000100:00000001:3.0:1775776827.741550:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776827.741552:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.741553:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776827.741557:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776827/real 1775776827] req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776827.741564:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776827.741565:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776827.741567:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776827.741569:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776827.741570:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776827.741572:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776827.741575:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776827.742022:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742030:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776827.742032:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.742033:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.742035:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776827.742039:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742046:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776827.742049:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742055:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776827.742212:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.742214:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.742215:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.742217:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776827.742219:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776827.742222:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776827.742225:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776827.742227:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776827.742231:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776827.742239:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799680:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776827.742243:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776827.742246:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742252:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776827.742254:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.742255:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776827.742257:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776827.742259:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776827.742262:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799808:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776827.742264:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776827.742266:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776827.742268:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.742270:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776827.742273:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776827.742275:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776827.742278:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776827.742281:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776827.742284:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776827.742287:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099799808, portal 10 00000100:00000001:3.0:1775776827.742289:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776827.742290:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:3.0:1775776827.742295:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742301:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776827.742303:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776827.742305:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776827.742307:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099799808, offset 0 00000400:00000200:3.0:1775776827.742311:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776827.742315:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.742319:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776827.742321:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776827.742323:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776827.742326:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776827.742328:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776827.742330:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4479:4479:256:4294967295] 192.168.201.139@tcp LPNI seq info [4479:4479:8:4294967295] 00000400:00000200:3.0:1775776827.742334:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.742340:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776827.742344:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.742347:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776827.742350:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776827.742353:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776827.742357:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776827.742360:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776827.742363:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.742365:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776827.742368:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776827.742371:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776827.742374:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776827.742376:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776827.742378:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776827.742381:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742388:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.742390:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742396:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.742398:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776827.742400:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776827.742403:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776827.742406:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776827.742408:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313215495 00000100:00000001:3.0:1775776827.742410:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313216758 00000100:00000001:3.0:1775776827.742412:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313215495 00000100:00000001:3.0:1775776827.742414:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313216758 00000100:00000001:3.0:1775776827.742417:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776827.742418:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.742420:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776827.742423:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776827/real 1775776827] req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776827.742431:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776827.742432:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776827.742434:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776827.742436:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776827.742437:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776827.742439:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776827.742442:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776827.742450:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742456:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776827.742457:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.742459:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.742460:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776827.742463:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742470:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776827.742473:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742479:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776827.742481:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.742483:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.742484:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.742486:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776827.742487:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776827.742489:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776827.742492:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776827.742494:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776827.742498:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776827.742505:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799808:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776827.742509:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776827.742513:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742518:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776827.742520:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.742522:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776827.742523:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776827.742525:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776827.742527:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799936:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776827.742530:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776827.742532:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776827.742533:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.742535:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776827.742538:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776827.742540:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776827.742543:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776827.742546:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776827.742549:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776827.742552:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099799936, portal 4 00000100:00000001:3.0:1775776827.742553:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776827.742554:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:3.0:1775776827.742559:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.742747:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776827.742750:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776827.742753:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776827.742757:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099799936, offset 0 00000400:00000200:3.0:1775776827.742761:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776827.742816:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.742820:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776827.742883:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776827.742888:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776827.742890:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776827.742892:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776827.742896:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4480:4480:256:4294967295] 192.168.201.139@tcp LPNI seq info [4480:4480:8:4294967295] 00000400:00000200:3.0:1775776827.742899:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.742906:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776827.742910:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.742913:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776827.742917:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776827.742919:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776827.742923:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776827.742926:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776827.742929:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.742931:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776827.742934:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776827.742936:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776827.742939:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776827.742941:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776827.742944:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776827.742947:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743018:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.743022:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743028:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.743030:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776827.743033:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776827.743036:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776827.743040:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776827.743041:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313215505 00000100:00000001:3.0:1775776827.743044:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313216758 00000100:00000001:3.0:1775776827.743045:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313215505 00000100:00000001:3.0:1775776827.743047:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313216758 00000100:00000001:3.0:1775776827.743049:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776827.743051:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.743053:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776827.743056:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776827/real 1775776827] req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776827.743134:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776827.743135:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776827.743137:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776827.743140:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776827.743141:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776827.743143:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776827.743147:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776827.743218:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743224:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776827.743226:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.743228:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.743229:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776827.743232:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743239:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776827.743242:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743248:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776827.743250:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.743252:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.743253:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.743255:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776827.743256:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776827.743259:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776827.743410:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776827.743413:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776827.743418:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776827.743425:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099799936:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776827.743429:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776827.743432:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743438:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776827.743439:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.743441:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776827.743443:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776827.743445:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776827.743448:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800064:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776827.743451:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776827.743453:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776827.743454:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776827.743456:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776827.743459:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115b000. 02000000:00000001:3.0:1775776827.743461:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776827.743464:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776827.743466:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776827.743469:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776827.743471:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099800064, portal 4 00000100:00000001:3.0:1775776827.743473:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776827.743474:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:3.0:1775776827.743478:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743483:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776827.743485:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776827.743487:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776827.743489:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099800064, offset 0 00000400:00000200:3.0:1775776827.743493:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776827.743496:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.743499:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776827.743502:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776827.743504:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776827.743506:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776827.743508:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776827.743511:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4481:4481:256:4294967295] 192.168.201.139@tcp LPNI seq info [4481:4481:8:4294967295] 00000400:00000200:3.0:1775776827.743515:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776827.743521:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776827.743525:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.743527:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776827.743531:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776827.743533:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775776827.743537:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776827.743540:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1775776827.743543:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776827.743544:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776827.743547:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776827.743550:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776827.743553:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776827.743555:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776827.743557:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776827.743560:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743578:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.743580:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.743889:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.743891:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776827.743896:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776827.743899:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776827.743903:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776827.743905:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313215497 00000100:00000001:3.0:1775776827.743907:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313216758 00000100:00000001:3.0:1775776827.743909:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313215497 00000100:00000001:3.0:1775776827.743911:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313216758 00000100:00000001:3.0:1775776827.743913:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776827.743914:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.743916:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776827.743920:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776827/real 1775776827] req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776827.743928:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776827.743929:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776827.743931:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000400:00000200:3.0:1775776827.743933:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000400:00000010:3.0:1775776827.743935:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000100:00000001:3.0:1775776827.743937:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776827.743939:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776827.743947:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.744018:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776827.744020:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744022:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.744023:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776827.744027:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.744034:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776827.744037:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.744044:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776827.744046:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744047:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776827.744049:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744050:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776827.744052:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776827.744055:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776827.744058:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776827.744060:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776827.744135:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776827.744143:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800064:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776827.744147:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744149:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.744211:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dfb80 x1862033099799552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.744219:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776827.744221:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776827.744223:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775776827.744227:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776827.744228:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776827.744230:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775776827.744232:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776827.744234:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:3.0:1775776827.744237:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776827.744239:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776827.744241:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dfb80. 00000100:00000001:3.0:1775776827.744244:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776827.744245:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744247:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.744250:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de680 x1862033099799680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.744255:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776827.744257:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776827.744258:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776827.744260:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776827.744261:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776827.744262:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776827.744264:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776827.744265:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:3.0:1775776827.744267:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776827.744268:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776827.744270:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de680. 00000100:00000001:3.0:1775776827.744272:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776827.744273:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744274:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.744277:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099799808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.744283:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776827.744284:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776827.744285:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776827.744287:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776827.744288:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776827.744289:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776827.744291:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776827.744292:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:3.0:1775776827.744295:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776827.744296:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776827.744297:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:3.0:1775776827.744298:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776827.744299:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744301:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.744304:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099799936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.744314:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776827.744315:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776827.744316:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776827.744318:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776827.744319:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776827.744320:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776827.744322:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776827.744323:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:3.0:1775776827.744324:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776827.744325:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776827.744326:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:3.0:1775776827.744328:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776827.744329:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744331:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776827.744333:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099800064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776827.744339:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776827.744340:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776827.744342:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115b000. 02000000:00000001:3.0:1775776827.744344:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776827.744345:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776827.744345:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776827.744347:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776827.744349:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:3.0:1775776827.744351:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776827.744351:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776827.744352:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:3.0:1775776827.744355:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776827.744356:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744358:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776827.744364:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776827.744366:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776827.744368:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776827.744369:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776828.443916:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776828.443923:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776828.443924:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776828.443930:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776828.443933:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776828.443946:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776828.443948:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776828.443951:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776828.443954:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776828.507948:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776828.507955:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776828.507961:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776828.507963:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776828.507966:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776828.507967:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776828.507970:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776828.507971:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776828.508109:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776828.508114:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776828.508117:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776828.571895:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776828.571902:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776828.571904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776828.571909:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776828.571912:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776828.571916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776828.571918:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776828.571920:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776828.571923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776829.019870:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313216758, at 4313216768 00000100:00000010:1.0:1775776829.019886:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344c00. 00000100:00000040:0.0:1775776829.020140:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313216758, at 4313216768 00000100:00000010:0.0:1775776829.020147:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344600. 00000100:00000001:2.0:1775776829.467903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776829.467912:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776829.467914:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776829.467919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776829.467922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776829.467926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776829.467928:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776829.467931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776829.467934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776829.531873:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776829.531881:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776829.531885:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776829.531888:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776829.531893:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776829.531895:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776829.531897:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776829.531900:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776829.531904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776829.531905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776829.531908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776829.531909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776829.531912:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776829.531914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776829.531980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776829.531984:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776829.531987:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776829.596114:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776829.596121:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776829.596123:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776829.596128:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776829.596132:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776829.596136:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776829.596138:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776829.596140:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776829.596144:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776830.491898:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776830.491905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776830.491907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776830.491913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776830.491917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776830.491920:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776830.491922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776830.491924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776830.491927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776830.555911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776830.555920:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776830.555923:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776830.555926:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555929:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776830.555930:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555932:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555934:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776830.555939:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776830.555940:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776830.555941:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555942:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776830.555943:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555944:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555946:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776830.555951:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776830.555952:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776830.555953:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555955:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776830.555956:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555957:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776830.555958:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776830.556009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776830.556014:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776830.556018:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776830.619903:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776830.619910:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776830.619912:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776830.619917:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776830.619921:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776830.619924:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776830.619926:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776830.619929:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776830.619932:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776831.516763:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776831.516771:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776831.516773:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776831.516811:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776831.516816:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776831.516820:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776831.516822:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776831.516824:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776831.516827:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776831.579976:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776831.579983:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776831.579989:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776831.579991:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776831.579994:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776831.579995:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776831.579998:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776831.579999:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776831.580297:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776831.580300:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776831.580303:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776831.644134:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776831.644139:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776831.644140:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776831.644145:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776831.644147:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776831.644150:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776831.644152:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776831.644154:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776831.644156:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.540327:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776832.540333:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776832.540335:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776832.540339:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776832.540341:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776832.540344:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.540346:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776832.540348:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.540350:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776832.603899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776832.603904:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776832.603908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776832.603909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776832.603911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776832.603912:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776832.603914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776832.603914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776832.604055:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776832.604058:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776832.604060:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776832.667895:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776832.667902:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776832.667904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776832.667909:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776832.667913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776832.667917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776832.667919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776832.667922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776832.667925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776832.860136:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776832.860151:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776832.860155:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776832.860160:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776832.860166:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776832.860177:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18548 00000400:00000200:0.0:1775776832.860192:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776832.860200:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776832.860205:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776832.860288:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860292:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.860295:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860298:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776832.860303:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776832.860306:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776832.860309:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776832.860313:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776832.860315:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776832.860317:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776832.860320:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860322:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.860324:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860326:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776832.860330:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776832.860334:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860337:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776832.860342:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776832.860346:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776832.860350:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.860353:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776832.860357:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776832.860369:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776832.860374:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776832.860379:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776832.860381:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.860385:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776832.860388:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776832.860394:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776832.860405:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776832.860413:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776832.860423:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776832.860509:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776832.860515:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776832.860527:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776832.860629:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776832.860634:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776832.860637:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776832.860643:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776832.860652:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776832.860653:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776832.860658:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776832.860661:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776832.860664:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776832.860668:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099800192:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775776832.860672:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776832.860677:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776832.860680:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776832.860682:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776832.860691:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115b000. 02000000:00000001:3.0:1775776832.860693:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776832.860701:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776832.860707:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d528. 00000400:00000010:3.0:1775776832.860714:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1088. 00000100:00000200:3.0:1775776832.860721:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099800192, portal 25 00000100:00000001:3.0:1775776832.860722:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776832.860724:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:3.0:1775776832.860728:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776832.860733:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776832.860735:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776832.860738:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776832.860741:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099800192, offset 0 00000400:00000200:3.0:1775776832.860748:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776832.860756:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776832.860764:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776832.860768:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776832.860772:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776832.860776:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776832.860815:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776832.860818:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4482:4482:256:4294967295] 192.168.201.139@tcp LPNI seq info [4482:4482:8:4294967295] 00000400:00000200:3.0:1775776832.860822:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776832.860830:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776832.860836:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776832.860840:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776832.860846:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776832.860849:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775776832.860853:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776832.860857:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1775776832.860873:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776832.860875:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776832.860880:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776832.860884:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776832.860888:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776832.860889:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776832.860893:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776832.860896:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776832.860931:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776832.860935:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776832.860938:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776832.860940:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776832.860942:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776832.860945:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776832.860948:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18548 00000400:00000200:0.0:1775776832.860953:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776832.860956:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776832.860959:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776832.860962:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860965:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.860968:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860970:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776832.860974:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776832.860976:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776832.860979:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776832.860982:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776832.860984:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776832.860986:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776832.860989:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860991:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.860993:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.860994:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776832.860997:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776832.861000:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861003:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776832.861023:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776832.861030:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776832.861032:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.861034:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776832.861039:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776832.861042:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.861043:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776832.861049:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776832.861053:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776832.861056:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776832.861058:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.861061:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776832.861064:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776832.861067:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776832.861073:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776832.861079:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776832.861086:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776832.861094:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776832.861096:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776832.861100:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776832.861208:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776832.861214:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776832.861219:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776832.861220:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776832.861225:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776832.861228:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776832.861230:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313221878 00000100:00000001:0.0:1775776832.861285:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776832.861289:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776832.861292:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776832.861293:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776832.861295:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776832.861297:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776832.861300:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18548 00000100:00000010:2.0:1775776832.861302:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b8a0. 00000400:00000200:0.0:1775776832.861305:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776832.861307:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776832.861309:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776832.861310:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775776832.861312:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861314:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.861316:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861318:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776832.861318:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313221878 00000020:00000001:0.0:1775776832.861320:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776832.861321:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000100:00000010:2.0:1775776832.861321:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b3c0. 00000020:00000001:0.0:1775776832.861323:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776832.861324:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000020:00000040:0.0:1775776832.861325:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776832.861326:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000001:2.0:1775776832.861327:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775776832.861328:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000001:2.0:1775776832.861329:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776832.861331:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776832.861332:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775776832.861333:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.861335:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861336:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00100000:2.0:1775776832.861336:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776832/real 1775776832] req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776832.861339:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776832.861342:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861344:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776832.861346:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1775776832.861346:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1775776832.861348:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1775776832.861348:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1775776832.861350:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776832.861350:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d528. 00000100:00000001:0.0:1775776832.861352:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775776832.861353:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1088 00000008:00100000:0.0:1775776832.861354:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000400:00000010:2.0:1775776832.861355:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1088. 00000008:00000001:0.0:1775776832.861357:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.861358:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861358:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776832.861361:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776832.861362:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776832.861364:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776832.861366:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776832.861368:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.861369:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1775776832.861370:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776832.861371:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776832.861373:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776832.861376:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:2.0:1775776832.861377:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776832.861378:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776832.861379:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861381:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.861382:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776832.861383:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775776832.861386:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776832.861390:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776832.861392:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861393:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776832.861395:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776832.861397:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776832.861403:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861404:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776832.861405:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776832.861407:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776832.861408:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776832.861409:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776832.861410:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776832.861410:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776832.861411:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776832.861412:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776832.861413:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776832.861414:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776832.861416:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18548 00000100:00080000:2.0:1775776832.861416:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:0.0:1775776832.861419:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:2.0:1775776832.861420:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000400:00000200:0.0:1775776832.861421:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:2.0:1775776832.861422:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1775776832.861423:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776832.861426:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000040:2.0:1775776832.861426:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776832.861428:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.861430:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861432:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776832.861434:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00100000:2.0:1775776832.861434:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099800192:192.168.201.139@tcp:250:kworker.0 00000020:00000040:0.0:1775776832.861435:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776832.861436:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776832.861438:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776832.861439:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776832.861439:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.861440:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1775776832.861441:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776832.861443:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000040:2.0:1775776832.861443:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099800192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776832.861445:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.861447:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861449:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776832.861450:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000040:0.0:1775776832.861451:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 02000000:00000001:2.0:1775776832.861452:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1775776832.861454:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 02000000:00000010:2.0:1775776832.861455:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115b000. 00000100:00000001:0.0:1775776832.861456:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776832.861457:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776832.861457:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776832.861458:0:301888:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:0.0:1775776832.861459:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776832.861460:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776832.861460:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861462:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000020:00000001:2.0:1775776832.861462:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 00000008:00100000:0.0:1775776832.861464:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 02000000:00000010:2.0:1775776832.861464:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759dc00. 00000008:00000001:0.0:1775776832.861466:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776832.861466:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1775776832.861468:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776832.861468:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776832.861470:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000100:00000010:2.0:1775776832.861470:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776832.861472:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00000001:2.0:1775776832.861473:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776832.861474:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775776832.861475:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776832.861476:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861476:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776832.861478:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776832.861479:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776832.861480:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776832.861483:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:2.0:1775776832.861483:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776832.861484:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.861485:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861487:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776832.861488:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1775776832.861489:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776832.861489:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776832.861492:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776832.861496:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776832.861498:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861499:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776832.861500:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776832.861501:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776832.861502:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776832.861504:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776832.861506:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.861508:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776832.861509:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800320:192.168.201.139@tcp:38:kworker.0 00000100:00080000:0.0:1775776832.861511:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776832.861512:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1775776832.861514:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:2.0:1775776832.861514:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776832.861515:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1775776832.861515:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776832.861516:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1775776832.861517:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776832.861518:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776832.861520:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18548 02000000:00000010:2.0:1775776832.861520:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776832.861522:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776832.861523:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776832.861525:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776832.861526:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000010:2.0:1775776832.861527:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000100:00080000:0.0:1775776832.861528:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776832.861530:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000010:2.0:1775776832.861531:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000001:0.0:1775776832.861532:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776832.861534:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000200:2.0:1775776832.861534:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099800320, portal 10 00000100:00000001:0.0:1775776832.861536:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776832.861536:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776832.861537:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000020:00000001:0.0:1775776832.861538:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776832.861539:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776832.861540:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776832.861541:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776832.861543:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776832.861543:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776832.861546:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776832.861549:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776832.861550:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776832.861551:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776832.861552:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00000040:0.0:1775776832.861553:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000010:2.0:1775776832.861554:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000001:0.0:1775776832.861555:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776832.861557:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00000200:2.0:1775776832.861557:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099800320, offset 0 00000100:00080000:0.0:1775776832.861559:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776832.861561:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776832.861561:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 02000000:00000001:0.0:1775776832.861563:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775776832.861564:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:0.0:1775776832.861565:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776832.861567:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776832.861568:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:0.0:1775776832.861569:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776832.861571:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000400:00000200:2.0:1775776832.861571:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000001:0.0:1775776832.861573:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776832.861573:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000001:0.0:1775776832.861574:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776832.861576:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000010:0.0:1775776832.861577:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000400:00000200:2.0:1775776832.861578:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000020:00000040:0.0:1775776832.861579:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000400:00000200:2.0:1775776832.861581:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4483:4483:256:4294967295] 192.168.201.139@tcp LPNI seq info [4483:4483:8:4294967295] 02000000:00000001:0.0:1775776832.861582:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776832.861583:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776832.861585:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1775776832.861585:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:0.0:1775776832.861586:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776832.861587:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776832.861590:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776832.861593:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776832.861597:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776832.861604:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776832.861606:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776832.861609:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776832.861614:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000001:0.0:1775776832.861616:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776832.861620:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18558) 00000800:00000200:2.0:1775776832.861676:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776832.861686:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776832.861689:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776832.861699:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776832.861719:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776832.861728:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1775776832.861731:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776832.861754:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776832.861759:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776832.861762:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776832.861765:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776832.861767:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776832.861770:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776832.861773:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.861803:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776832.861805:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.861810:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861810:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776832.861814:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776832.861816:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776832.861819:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776832.861827:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313220614 00000100:00000001:2.0:1775776832.861829:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313221878 00000100:00000001:2.0:1775776832.861831:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313220614 00000100:00000001:2.0:1775776832.861832:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313221878 00000100:00000001:2.0:1775776832.861833:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776832.861835:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.861836:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776832.861838:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776832/real 1775776832] req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776832.861856:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776832.861858:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776832.861859:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776832.861861:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776832.861862:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776832.861863:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776832.861876:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776832.861881:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.861885:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776832.861886:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.861887:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.861888:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776832.861889:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.861893:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776832.861895:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.861898:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776832.861900:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.861901:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.861902:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.861903:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776832.861905:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776832.861906:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776832.861908:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776832.861909:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776832.861911:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776832.861916:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800320:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776832.861918:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776832.861919:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.861923:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776832.861924:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776832.861925:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776832.861926:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776832.861927:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776832.861928:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800448:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776832.861930:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776832.861931:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776832.861932:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776832.861933:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776832.861936:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776832.861937:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776832.861939:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776832.861940:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776832.861942:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776832.861944:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099800448, portal 10 00000100:00000001:2.0:1775776832.861945:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776832.861946:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748096 : -114841967803520 : ffff978d454dc380) 00000100:00000040:2.0:1775776832.861950:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.861953:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776832.861955:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776832.861956:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776832.861958:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099800448, offset 0 00000400:00000200:2.0:1775776832.861960:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776832.861962:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776832.861964:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776832.861966:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776832.861968:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776832.861969:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776832.861971:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776832.861972:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4484:4484:256:4294967295] 192.168.201.139@tcp LPNI seq info [4484:4484:8:4294967295] 00000400:00000200:2.0:1775776832.861975:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776832.861979:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776832.861981:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776832.861983:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776832.861985:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776832.861987:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776832.861989:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776832.861991:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1775776832.861993:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776832.861994:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776832.861996:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776832.861998:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776832.862000:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776832.862001:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776832.862002:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776832.862004:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862088:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776832.862092:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862099:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.862100:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776832.862105:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776832.862108:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776832.862111:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776832.862113:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313220615 00000100:00000001:2.0:1775776832.862115:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313221878 00000100:00000001:2.0:1775776832.862117:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313220615 00000100:00000001:2.0:1775776832.862119:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313221878 00000100:00000001:2.0:1775776832.862121:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776832.862123:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.862125:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776832.862128:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776832/real 1775776832] req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776832.862136:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776832.862137:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776832.862138:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776832.862140:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776832.862141:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776832.862143:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776832.862144:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776832.862149:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862153:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776832.862154:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862155:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.862156:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776832.862158:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862162:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776832.862164:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862168:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776832.862169:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862170:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.862171:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862172:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776832.862173:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776832.862174:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776832.862176:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776832.862177:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776832.862179:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776832.862183:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800448:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776832.862185:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862186:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776832.862187:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099800320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862191:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776832.862192:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776832.862193:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776832.862195:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776832.862196:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776832.862197:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776832.862198:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776832.862199:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d000. 02000000:00000001:2.0:1775776832.862201:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776832.862201:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776832.862202:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:2.0:1775776832.862204:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776832.862205:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862206:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776832.862207:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc380 x1862033099800448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862211:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776832.862211:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776832.862212:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776832.862213:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776832.862214:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776832.862214:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776832.862215:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776832.862216:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:2.0:1775776832.862217:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776832.862217:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776832.862218:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc380. 00000100:00000001:2.0:1775776832.862219:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776832.862220:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862221:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862223:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776832.862224:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.862225:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776832.862226:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776832.862226:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776832.862228:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862231:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776832.862232:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776832.862233:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776832.862234:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776832.862235:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776832.862237:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800576:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776832.862239:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776832.862241:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776832.862241:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776832.862242:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776832.862245:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776832.862246:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776832.862248:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776832.862249:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776832.862252:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776832.862254:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099800576, portal 4 00000100:00000001:2.0:1775776832.862255:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776832.862255:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741750784 : -114841967800832 : ffff978d454dce00) 00000100:00000040:2.0:1775776832.862259:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862263:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776832.862264:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776832.862265:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776832.862267:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099800576, offset 0 00000400:00000200:2.0:1775776832.862269:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776832.862271:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776832.862273:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776832.862275:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776832.862277:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776832.862279:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776832.862280:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776832.862282:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4485:4485:256:4294967295] 192.168.201.139@tcp LPNI seq info [4485:4485:8:4294967295] 00000400:00000200:2.0:1775776832.862284:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776832.862288:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776832.862291:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776832.862293:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776832.862295:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776832.862297:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776832.862300:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776832.862301:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1775776832.862303:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776832.862304:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776832.862306:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776832.862308:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776832.862310:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776832.862311:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776832.862312:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776832.862314:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862318:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776832.862320:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862323:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.862324:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776832.862326:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776832.862327:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776832.862329:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776832.862330:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313220624 00000100:00000001:2.0:1775776832.862332:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313221878 00000100:00000001:2.0:1775776832.862333:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313220624 00000100:00000001:2.0:1775776832.862334:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313221878 00000100:00000001:2.0:1775776832.862335:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776832.862336:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.862337:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776832.862339:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776832/real 1775776832] req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776832.862343:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776832.862344:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776832.862346:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776832.862348:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776832.862349:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776832.862352:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776832.862354:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776832.862362:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862369:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776832.862370:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862371:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.862372:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776832.862374:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862379:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776832.862383:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862390:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776832.862392:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862394:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.862395:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862397:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776832.862398:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776832.862401:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776832.862404:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776832.862406:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776832.862410:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776832.862415:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800576:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776832.862418:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776832.862419:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862423:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776832.862424:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776832.862425:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776832.862426:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776832.862427:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776832.862428:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800704:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776832.862430:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776832.862431:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776832.862432:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776832.862433:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776832.862435:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776832.862437:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776832.862439:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776832.862440:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776832.862442:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776832.862444:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099800704, portal 4 00000100:00000001:2.0:1775776832.862445:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776832.862446:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748992 : -114841967802624 : ffff978d454dc700) 00000100:00000040:2.0:1775776832.862450:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862454:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776832.862455:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776832.862456:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776832.862457:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099800704, offset 0 00000400:00000200:2.0:1775776832.862459:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776832.862462:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776832.862464:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776832.862465:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776832.862467:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776832.862468:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776832.862470:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776832.862472:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4486:4486:256:4294967295] 192.168.201.139@tcp LPNI seq info [4486:4486:8:4294967295] 00000400:00000200:2.0:1775776832.862474:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776832.862477:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776832.862480:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776832.862481:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776832.862484:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776832.862485:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1775776832.862488:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776832.862489:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1775776832.862492:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776832.862493:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776832.862495:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776832.862497:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776832.862499:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776832.862500:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776832.862502:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776832.862505:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862512:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776832.862514:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862519:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.862521:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776832.862523:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776832.862526:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776832.862528:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776832.862530:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313220615 00000100:00000001:2.0:1775776832.862532:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313221878 00000100:00000001:2.0:1775776832.862535:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313220615 00000100:00000001:2.0:1775776832.862537:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313221878 00000100:00000001:2.0:1775776832.862539:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776832.862541:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.862542:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776832.862546:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776832/real 1775776832] req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776832.862553:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776832.862554:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776832.862556:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776832.862558:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776832.862559:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776832.862560:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776832.862562:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776832.862566:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862570:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776832.862571:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862572:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.862572:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776832.862574:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862578:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776832.862580:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862584:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776832.862585:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862586:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776832.862586:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862588:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776832.862589:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776832.862591:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776832.862593:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776832.862594:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776832.862596:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776832.862600:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800704:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776832.862602:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862603:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776832.862604:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dce00 x1862033099800576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862608:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776832.862609:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776832.862610:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776832.862612:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776832.862612:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776832.862613:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776832.862614:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776832.862615:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:2.0:1775776832.862617:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776832.862617:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776832.862618:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dce00. 00000100:00000001:2.0:1775776832.862620:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776832.862621:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862622:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776832.862623:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc700 x1862033099800704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776832.862626:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776832.862627:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776832.862628:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776832.862629:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776832.862630:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776832.862630:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776832.862632:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776832.862632:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:2.0:1775776832.862634:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776832.862635:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776832.862635:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc700. 00000100:00000001:2.0:1775776832.862636:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776832.862637:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862638:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776832.862640:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776832.862640:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776832.862641:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776832.862642:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776833.563952:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776833.563961:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776833.563964:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776833.563971:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776833.563975:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776833.563981:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776833.563984:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776833.563986:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776833.564078:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776833.628402:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776833.628410:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776833.628418:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776833.628419:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776833.628422:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776833.628424:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776833.628428:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776833.628429:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776833.628661:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776833.628665:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776833.628669:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776833.692868:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776833.692875:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776833.692877:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776833.692884:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776833.692888:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776833.692893:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776833.692895:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776833.692898:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776833.692901:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776834.140090:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313221878, at 4313221888 00000100:00000040:1.0:1775776834.140099:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313221878, at 4313221888 00000100:00000010:0.0:1775776834.140109:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000010:1.0:1775776834.140113:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000001:2.0:1775776834.587956:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776834.590169:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776834.590172:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776834.590180:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776834.590184:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776834.590189:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776834.590192:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776834.590195:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776834.590198:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776834.653841:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776834.653852:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776834.653861:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776834.653862:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776834.653866:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776834.653867:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776834.653870:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776834.653871:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776834.656853:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776834.656862:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776834.656870:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776834.715940:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776834.715949:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776834.715951:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776834.715958:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776834.715962:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776834.715968:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776834.715971:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776834.715984:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776834.715988:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776835.611976:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776835.611987:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776835.611989:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776835.611996:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776835.612422:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776835.612428:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776835.612486:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776835.612489:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776835.612493:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776835.678994:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776835.679002:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776835.679010:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776835.679011:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776835.679015:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776835.679016:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776835.679020:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776835.679021:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776835.679159:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776835.679163:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776835.679166:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776835.740880:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776835.740890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776835.740893:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776835.740899:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776835.740903:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776835.740908:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776835.740911:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776835.740914:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776835.740917:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776836.635952:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776836.635961:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776836.635963:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776836.635969:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776836.635974:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776836.635979:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776836.635982:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776836.635985:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776836.635988:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776836.700723:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776836.700731:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776836.700738:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776836.700740:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776836.700744:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776836.700745:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776836.700748:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776836.700750:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776836.704358:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776836.704366:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776836.704371:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776836.764171:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776836.764181:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776836.764184:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776836.764193:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776836.764198:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776836.764204:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776836.764207:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776836.764210:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776836.764214:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.659999:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776837.660008:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776837.660011:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776837.660019:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776837.660024:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776837.660030:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776837.660034:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776837.660037:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776837.660041:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776837.723968:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776837.723976:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776837.723985:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776837.723986:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776837.723990:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776837.723991:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776837.723995:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776837.723996:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776837.724303:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776837.724307:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776837.724311:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776837.788060:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776837.788070:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776837.788073:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776837.788081:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776837.788085:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776837.788091:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776837.788094:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776837.788097:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776837.788111:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776837.980018:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776837.980031:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776837.980034:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776837.980039:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776837.980045:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776837.980054:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18553 00000400:00000200:2.0:1775776837.980068:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776837.980076:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776837.980080:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776837.980084:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980088:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.980092:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980094:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776837.980100:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776837.980102:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776837.980105:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776837.980109:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776837.980112:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776837.980115:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776837.980119:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980121:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.980123:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980125:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776837.980128:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776837.980132:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980134:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776837.980138:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776837.980141:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776837.980144:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.980146:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776837.980149:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776837.980163:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d500. 00000020:00000040:2.0:1775776837.980166:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776837.980171:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776837.980173:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.980177:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776837.980180:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776837.980185:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776837.980195:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507400. 00000100:00000001:2.0:1775776837.980202:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776837.980213:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776837.980225:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776837.980231:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776837.980249:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000855db814] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.980301:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776837.980306:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776837.980310:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776837.980316:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.980325:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776837.980327:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.980331:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776837.980334:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776837.980337:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776837.980341:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099800832:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776837.980346:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776837.980352:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776837.980354:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.980358:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776837.980364:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776837.980367:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776837.980373:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776837.980378:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776837.980386:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776837.980395:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099800832, portal 25 00000100:00000001:0.0:1775776837.980397:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776837.980399:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716032 : -114841420835584 : ffff978d65e7d500) 00000100:00000040:0.0:1775776837.980405:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.980413:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776837.980416:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776837.980420:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776837.980422:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099800832, offset 0 00000400:00000200:0.0:1775776837.980429:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776837.980438:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.980444:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776837.980450:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776837.980456:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776837.980461:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776837.980464:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776837.980470:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4487:4487:256:4294967295] 192.168.201.139@tcp LPNI seq info [4487:4487:8:4294967295] 00000400:00000200:0.0:1775776837.980475:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.980488:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776837.980494:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.980500:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776837.980507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776837.980511:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776837.980517:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776837.980521:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776837.980526:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.980529:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776837.980537:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776837.980542:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776837.980548:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776837.980550:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776837.980555:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776837.980570:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776837.980662:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776837.980667:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776837.980671:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776837.980672:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776837.980674:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776837.980677:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776837.980680:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18553 00000400:00000200:2.0:1775776837.980684:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776837.980687:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776837.980689:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776837.980692:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980694:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.980696:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980699:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776837.980701:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776837.980702:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776837.980703:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776837.980705:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776837.980707:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776837.980709:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776837.980711:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980713:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.980715:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980717:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776837.980720:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776837.980723:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776837.980727:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776837.980728:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776837.980736:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776837.980738:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.980740:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776837.980746:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776837.980749:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.980751:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776837.980756:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f100. 00000020:00000040:2.0:1775776837.980758:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776837.980761:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776837.980762:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.980764:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776837.980766:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776837.980768:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776837.980772:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506400. 00000100:00000001:2.0:1775776837.980776:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776837.980828:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776837.980837:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776837.980841:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776837.980845:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048605922] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.982474:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.982493:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.982506:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.982508:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776837.982526:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776837.982534:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776837.982538:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313226998 00000100:00000010:0.0:1775776837.982551:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776837.982568:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776837.982579:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313226998 00000100:00000010:0.0:1775776837.982582:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237900. 00000100:00000001:0.0:1775776837.982586:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776837.982590:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776837.982594:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.982597:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776837.982603:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776837/real 1775776837] req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776837.982617:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776837.982619:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776837.982623:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776837.982627:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776837.982630:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776837.982633:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776837.982637:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776837.982647:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.982655:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776837.982656:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.982659:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.982660:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776837.982664:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.982671:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776837.982675:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.982690:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776837.982693:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.982695:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.982696:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.982699:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776837.982701:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776837.982704:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776837.982709:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776837.982711:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776837.982715:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776837.982723:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099800832:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776837.982729:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.982732:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.982735:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d500 x1862033099800832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.982742:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776837.982746:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776837.982750:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776837.982754:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776837.982757:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776837.982759:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776837.982762:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776837.982764:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507400. 02000000:00000001:0.0:1775776837.982769:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776837.982770:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776837.982773:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d500. 00000100:00000001:0.0:1775776837.982777:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776837.982792:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.982795:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.982805:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776837.982807:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.982810:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776837.982813:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776837.982815:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776837.982821:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.982829:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776837.982831:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.982835:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776837.982839:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776837.982842:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776837.982845:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800960:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776837.982849:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776837.982854:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776837.982857:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.982859:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776837.982864:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776837.982866:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776837.982871:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776837.982875:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776837.982881:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776837.982887:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099800960, portal 10 00000100:00000001:0.0:1775776837.982889:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776837.982891:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288723200 : -114841420828416 : ffff978d65e7f100) 00000100:00000040:0.0:1775776837.982899:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.982906:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776837.982908:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776837.982911:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776837.982914:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099800960, offset 0 00000400:00000200:0.0:1775776837.982930:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776837.982938:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.982947:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776837.982954:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776837.982959:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776837.982963:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776837.982966:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776837.982971:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4488:4488:256:4294967295] 192.168.201.139@tcp LPNI seq info [4488:4488:8:4294967295] 00000400:00000200:0.0:1775776837.982975:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.982985:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776837.982991:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.982997:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776837.983004:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776837.983008:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776837.983015:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776837.983018:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776837.983023:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.983025:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776837.983031:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776837.983036:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776837.983040:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776837.983042:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776837.983045:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776837.983049:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.983058:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.983060:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.983067:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.983069:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776837.983071:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776837.983075:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776837.983078:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776837.983080:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313225733 00000100:00000001:0.0:1775776837.983083:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313226998 00000100:00000001:0.0:1775776837.983085:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313225733 00000100:00000001:0.0:1775776837.983087:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313226998 00000100:00000001:0.0:1775776837.983090:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776837.983091:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.983093:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776837.983096:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776837/real 1775776837] req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776837.983103:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776837.983105:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776837.983106:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776837.983108:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776837.983110:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776837.983112:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776837.983114:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776837.983123:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.983130:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776837.983132:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.983133:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.983135:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776837.983138:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.983145:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776837.983149:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.983155:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776837.983158:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.983159:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.983161:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.983163:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776837.983165:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776837.983168:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776837.983171:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776837.983173:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776837.983176:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776837.983184:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099800960:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776837.983188:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.983190:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.983192:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f100 x1862033099800960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.983198:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776837.983199:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776837.983201:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776837.983203:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776837.983204:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776837.983206:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776837.983207:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776837.983209:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506400. 02000000:00000001:0.0:1775776837.983211:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776837.983212:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776837.983213:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f100. 00000100:00000001:0.0:1775776837.983215:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776837.983216:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.983218:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.983222:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776837.983224:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.983226:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776837.983228:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.983544:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776837.983551:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776837.983621:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776837.983624:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776837.983626:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776837.983630:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776837.983635:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18553 00000400:00000200:2.0:1775776837.983641:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776837.983646:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776837.983650:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776837.983654:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983657:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.983659:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983662:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776837.983665:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776837.983667:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776837.983670:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776837.983673:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776837.983675:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776837.983678:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776837.983681:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983683:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.983685:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983687:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776837.983690:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776837.983693:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983695:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776837.983697:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776837.983704:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776837.983706:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.983708:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776837.983712:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776837.983715:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.983717:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776837.983723:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f480. 00000020:00000040:2.0:1775776837.983726:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776837.983730:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776837.983732:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.983735:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776837.983738:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776837.983742:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776837.983749:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504400. 00000100:00000001:2.0:1775776837.983754:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776837.983760:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776837.983770:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776837.983776:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776837.983797:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002c91b8b4] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776837.983809:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776837.983811:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776837.983815:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776837.983817:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776837.983818:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776837.983821:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776837.983823:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18553 00000400:00000200:2.0:1775776837.983826:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776837.983829:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776837.983831:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776837.983833:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983835:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.983837:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983839:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776837.983841:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776837.983842:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776837.983844:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776837.983846:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776837.983848:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776837.983850:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776837.983853:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983855:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.983857:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983859:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776837.983861:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776837.983863:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776837.983865:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776837.983866:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776837.983869:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776837.983871:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.983872:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776837.983875:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776837.983877:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.983878:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776837.983881:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d880. 00000020:00000040:2.0:1775776837.983883:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776837.983885:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776837.983887:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.983889:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776837.983890:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776837.983892:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776837.983898:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506400. 00000100:00000001:0.0:1775776837.984101:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776837.984104:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776837.984105:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776837.984112:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.984120:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776837.984121:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.984124:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776837.984126:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776837.984129:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776837.984132:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801088:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776837.984136:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776837.984138:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776837.984140:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.984142:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776837.984146:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776837.984148:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776837.984152:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776837.984155:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776837.984160:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776837.984165:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099801088, portal 10 00000100:00000001:0.0:1775776837.984167:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776837.984168:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724096 : -114841420827520 : ffff978d65e7f480) 00000100:00000040:0.0:1775776837.984175:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.984182:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776837.984185:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776837.984187:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776837.984189:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099801088, offset 0 00000400:00000200:0.0:1775776837.984195:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776837.984200:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.984205:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776837.984209:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776837.984212:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776837.984215:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776837.984217:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776837.984221:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4489:4489:256:4294967295] 192.168.201.139@tcp LPNI seq info [4489:4489:8:4294967295] 00000400:00000200:0.0:1775776837.984225:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.984233:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000001:2.0:1775776837.984324:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776837.984330:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776837.984338:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776837.984341:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776837.984345:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f88f35f] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776837.984354:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776837.984356:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776837.984360:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776837.984362:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776837.984363:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776837.984366:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776837.984368:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18553 00000400:00000200:2.0:1775776837.984373:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776837.984376:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776837.984378:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776837.984381:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.984383:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.984385:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.984388:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776837.984390:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776837.984391:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776837.984393:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776837.984395:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776837.984397:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776837.984399:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776837.984402:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.984403:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776837.984405:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776837.984407:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776837.984409:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776837.984412:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776837.984414:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776837.984415:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776837.984418:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776837.984420:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.984422:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776837.984424:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776837.984427:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.984428:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776837.984432:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7dc00. 00000020:00000040:2.0:1775776837.984434:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776837.984437:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776837.984438:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776837.984440:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776837.984441:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776837.984443:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776837.984447:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507400. 00000100:00000001:2.0:1775776837.984450:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776837.984454:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776837.984462:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776837.984464:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776837.984467:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f1188f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776837.984475:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776837.984481:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18563) 00000800:00000200:0.0:1775776837.984532:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.984536:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776837.984542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776837.984545:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776837.984551:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776837.984554:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776837.984558:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.984848:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776837.984855:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776837.984861:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776837.984866:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776837.984869:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776837.984872:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776837.984876:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.984886:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.984888:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.984896:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.984898:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776837.984904:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776837.984908:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776837.984912:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776837.984914:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313225738 00000100:00000001:0.0:1775776837.984929:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313226998 00000100:00000001:0.0:1775776837.984931:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313225738 00000100:00000001:0.0:1775776837.984934:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313226998 00000100:00000001:0.0:1775776837.984936:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776837.984939:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.984941:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776837.984945:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776837/real 1775776837] req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776837.984955:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776837.984957:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776837.984959:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776837.984962:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776837.984964:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776837.984966:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776837.984969:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776837.984978:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.984984:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776837.984985:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.984987:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.984989:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776837.984992:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.984998:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776837.985002:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.985009:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776837.985011:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.985013:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.985014:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.985016:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776837.985019:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776837.985021:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776837.985025:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776837.985027:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776837.985031:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776837.985040:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801088:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776837.985044:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.985046:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.985048:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f480 x1862033099801088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.985053:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776837.985055:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776837.985058:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776837.985060:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776837.985062:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776837.985063:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776837.985065:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776837.985067:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504400. 02000000:00000001:0.0:1775776837.985071:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776837.985072:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776837.985073:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f480. 00000100:00000001:0.0:1775776837.985075:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776837.985076:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.985078:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.985084:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776837.985085:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.985087:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776837.985089:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776837.985090:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776837.985094:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.985101:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776837.985102:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.985104:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776837.985106:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776837.985109:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776837.985112:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801216:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776837.985115:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776837.985117:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776837.985119:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.985120:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776837.985124:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776837.985126:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776837.985129:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776837.985132:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776837.985137:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776837.985139:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099801216, portal 4 00000100:00000001:0.0:1775776837.985141:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776837.985143:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716928 : -114841420834688 : ffff978d65e7d880) 00000100:00000040:0.0:1775776837.985149:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.985156:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776837.985159:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776837.985161:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776837.985163:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099801216, offset 0 00000400:00000200:0.0:1775776837.985168:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776837.985172:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.985176:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776837.985180:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776837.985183:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776837.985185:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776837.985188:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776837.985191:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4490:4490:256:4294967295] 192.168.201.139@tcp LPNI seq info [4490:4490:8:4294967295] 00000400:00000200:0.0:1775776837.985196:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.985203:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776837.985208:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.985211:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776837.985215:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776837.985218:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776837.985222:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776837.985225:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776837.985229:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.985231:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776837.985234:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776837.986390:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776837.986397:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776837.986400:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776837.986405:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776837.986411:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986424:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.986427:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986435:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.986437:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776837.986443:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776837.986447:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776837.986452:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776837.986455:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313225748 00000100:00000001:0.0:1775776837.986459:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313226998 00000100:00000001:0.0:1775776837.986462:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313225748 00000100:00000001:0.0:1775776837.986464:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313226998 00000100:00000001:0.0:1775776837.986466:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776837.986469:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.986472:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776837.986476:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776837/real 1775776837] req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776837.986486:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776837.986488:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776837.986490:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776837.986493:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776837.986495:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776837.986497:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776837.986500:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776837.986509:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986517:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776837.986519:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.986521:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.986523:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776837.986526:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986533:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776837.986537:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986545:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776837.986548:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.986549:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.986550:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.986553:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776837.986555:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776837.986558:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776837.986729:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776837.986732:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776837.986737:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776837.986744:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801216:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776837.986749:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776837.986753:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986759:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776837.986761:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.986763:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776837.986767:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776837.986769:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776837.986773:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801344:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776837.986776:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776837.986798:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776837.986801:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776837.986803:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776837.986812:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776837.986814:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776837.986819:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776837.986823:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776837.986827:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776837.986833:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099801344, portal 4 00000100:00000001:0.0:1775776837.986836:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776837.986837:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288717824 : -114841420833792 : ffff978d65e7dc00) 00000100:00000040:0.0:1775776837.986844:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986850:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776837.986854:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776837.986856:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776837.986858:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099801344, offset 0 00000400:00000200:0.0:1775776837.986863:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776837.986869:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.986875:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776837.986880:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776837.986883:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776837.986887:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776837.986889:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776837.986893:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4491:4491:256:4294967295] 192.168.201.139@tcp LPNI seq info [4491:4491:8:4294967295] 00000400:00000200:0.0:1775776837.986897:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776837.986905:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776837.986911:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.986925:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776837.986932:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776837.986934:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776837.986939:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776837.986941:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776837.986945:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776837.986947:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776837.986951:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776837.986955:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776837.986958:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:0.0:1775776837.986960:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:0.0:1775776837.986963:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776837.986966:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986974:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.986977:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.986984:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.986986:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776837.986989:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776837.986993:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776837.986996:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776837.986998:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313225740 00000100:00000001:0.0:1775776837.987000:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313226998 00000100:00000001:0.0:1775776837.987003:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313225740 00000100:00000001:0.0:1775776837.987005:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313226998 00000100:00000001:0.0:1775776837.987007:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776837.987009:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.987011:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776837.987014:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776837/real 1775776837] req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776837.987022:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776837.987023:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776837.987024:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776837.987026:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776837.987028:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776837.987030:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776837.987033:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776837.987041:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.987045:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776837.987047:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.987048:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.987049:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776837.987052:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.987057:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776837.987060:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.987066:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776837.987068:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.987070:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776837.987071:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.987072:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776837.987074:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776837.987076:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776837.987080:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776837.987081:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776837.987085:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776837.987092:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801344:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776837.987095:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.987097:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.987100:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d880 x1862033099801216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.987106:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776837.987108:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776837.987110:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776837.987113:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776837.987115:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776837.987117:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776837.987119:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776837.987121:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506400. 02000000:00000001:0.0:1775776837.987125:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776837.987127:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776837.987129:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d880. 00000100:00000001:0.0:1775776837.987132:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776837.987133:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.987135:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776837.987139:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7dc00 x1862033099801344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776862 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776837.987146:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776837.987147:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776837.987148:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776837.987150:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776837.987151:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776837.987152:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776837.987154:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776837.987155:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507400. 02000000:00000001:0.0:1775776837.987157:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776837.987159:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776837.987160:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7dc00. 00000100:00000001:0.0:1775776837.987161:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776837.987163:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.987165:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776837.987172:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776837.987174:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776837.987176:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776837.987178:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776838.684131:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776838.684142:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776838.684144:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776838.684151:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776838.684156:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776838.684160:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776838.684163:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776838.684166:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776838.684169:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776838.747975:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776838.747985:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776838.747988:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776838.747992:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776838.747998:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776838.748000:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776838.748014:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776838.748016:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776838.748021:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776838.748022:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776838.748026:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776838.748027:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776838.748031:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776838.748032:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776838.748174:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776838.748179:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776838.748182:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776838.813667:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776838.813676:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776838.813678:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776838.813685:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776838.813689:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776838.813695:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776838.813698:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776838.813700:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776838.813704:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776839.260123:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313226998, at 4313227008 00000100:00000010:1.0:1775776839.260142:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237900. 00000100:00000040:0.0:1775776839.260222:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313226998, at 4313227008 00000100:00000010:0.0:1775776839.260228:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:2.0:1775776839.707937:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776839.707945:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776839.707947:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776839.707954:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776839.707960:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776839.707965:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776839.707968:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776839.707973:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776839.707977:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776839.775038:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776839.775044:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776839.775048:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776839.775050:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776839.775052:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776839.775053:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776839.775056:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776839.775057:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776839.775165:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776839.775169:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776839.775172:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776839.863045:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776839.863053:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776839.863055:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776839.863061:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776839.863064:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776839.863069:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776839.863072:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776839.863074:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776839.863077:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776840.731917:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776840.731924:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776840.731926:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776840.731931:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776840.731935:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776840.731939:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776840.731942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776840.731944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776840.731947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796017:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776840.796022:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796027:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776840.796030:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776840.796033:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776840.796037:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796039:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776840.796042:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796044:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796047:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796051:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776840.796052:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776840.796053:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776840.796054:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796056:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776840.796058:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796059:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796061:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796064:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776840.796065:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776840.796067:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776840.796068:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796069:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776840.796071:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796072:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776840.796074:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776840.796189:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776840.796192:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776840.796195:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776840.923901:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776840.923909:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776840.923911:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776840.923916:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776840.923920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776840.923933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776840.923936:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776840.923938:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776840.923942:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776841.755885:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776841.755893:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776841.755895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776841.755900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776841.755905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776841.755919:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776841.755922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776841.755924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776841.755927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776841.820310:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776841.820316:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776841.820322:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776841.820323:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776841.820326:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776841.820327:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776841.820330:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776841.820332:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776841.820645:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776841.820649:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776841.820652:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776841.948814:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776841.948821:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776841.948822:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776841.948828:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776841.948832:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776841.948836:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776841.948838:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776841.948840:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776841.948843:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776842.780693:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776842.780700:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776842.780702:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776842.780707:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776842.780711:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776842.780715:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776842.780718:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776842.780734:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776842.780738:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776842.843879:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776842.843885:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776842.843890:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776842.843892:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776842.843895:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776842.843896:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776842.843900:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776842.843902:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776842.846368:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776842.846373:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776842.846377:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776842.972018:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776842.972024:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776842.972026:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776842.972032:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776842.972035:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776842.972039:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776842.972042:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776842.972044:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776842.972047:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776843.101049:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776843.101063:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776843.101066:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776843.101069:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776843.101072:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776843.101078:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18558 00000400:00000200:3.0:1775776843.101087:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776843.101091:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776843.101094:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776843.101096:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101098:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101100:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101101:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776843.101104:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776843.101105:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776843.101107:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776843.101109:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776843.101110:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776843.101111:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776843.101113:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101114:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101116:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101117:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776843.101120:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776843.101123:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101126:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776843.101128:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776843.101131:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776843.101134:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101136:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776843.101139:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776843.101149:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776843.101152:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776843.101156:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776843.101157:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101170:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776843.101173:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776843.101176:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776843.101185:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776843.101192:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776843.101201:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776843.101211:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776843.101215:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776843.101220:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776843.101245:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776843.101248:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776843.101251:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776843.101255:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776843.101256:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776843.101258:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776843.101261:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18558 00000400:00000200:3.0:1775776843.101265:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776843.101267:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776843.101269:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776843.101271:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101273:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101275:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101277:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776843.101278:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776843.101279:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776843.101281:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776843.101282:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776843.101284:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776843.101285:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776843.101288:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101290:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101291:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101293:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776843.101295:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776843.101297:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101299:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776843.101301:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776843.101306:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776843.101308:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101310:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776843.101314:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776843.101317:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101318:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776843.101322:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775776843.101324:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776843.101326:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776843.101327:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101328:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776843.101329:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776843.101330:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776843.101334:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776843.101337:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776843.101341:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776843.101347:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776843.101349:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776843.101352:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776843.101359:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776843.101361:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776843.101364:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776843.101366:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776843.101367:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776843.101369:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776843.101372:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18558 00000400:00000200:3.0:1775776843.101376:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776843.101378:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776843.101380:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776843.101382:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101384:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101386:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101388:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776843.101389:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776843.101390:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776843.101392:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776843.101393:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776843.101395:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776843.101397:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776843.101399:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101401:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101402:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101404:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776843.101406:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776843.101408:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101410:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776843.101412:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776843.101414:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776843.101416:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101418:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776843.101420:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776843.101422:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101424:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776843.101428:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776843.101430:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776843.101432:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776843.101433:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101435:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776843.101436:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776843.101437:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776843.101441:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775776843.101443:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776843.101448:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776843.101455:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776843.101457:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776843.101461:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776843.101469:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776843.101471:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776843.101474:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776843.101476:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776843.101477:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776843.101491:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776843.101494:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18558 00000400:00000200:3.0:1775776843.101498:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776843.101501:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776843.101503:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776843.101506:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101508:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101510:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101511:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776843.101513:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776843.101515:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776843.101517:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776843.101518:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776843.101520:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776843.101523:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776843.101525:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101527:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101529:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101530:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776843.101532:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776843.101535:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101537:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776843.101538:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776843.101541:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776843.101543:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101545:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776843.101547:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776843.101549:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101551:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776843.101554:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776843.101556:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776843.101558:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776843.101559:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101561:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776843.101562:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776843.101564:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776843.101568:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776843.101571:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776843.101575:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776843.101580:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776843.101582:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776843.101584:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776843.101589:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776843.101591:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776843.101593:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776843.101594:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776843.101595:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776843.101597:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776843.101598:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18558 00000400:00000200:3.0:1775776843.101600:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776843.101602:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776843.101603:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776843.101605:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101606:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101607:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101609:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776843.101610:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776843.101611:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776843.101612:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776843.101613:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776843.101614:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776843.101615:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776843.101617:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101618:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776843.101620:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101621:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776843.101623:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776843.101625:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776843.101627:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776843.101629:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776843.101631:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776843.101633:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101635:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776843.101637:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776843.101639:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101641:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776843.101644:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776843.101646:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776843.101649:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776843.101650:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776843.101651:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776843.101653:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776843.101654:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776843.101657:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776843.101661:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776843.101665:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776843.101672:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776843.101674:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776843.101677:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776843.101684:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776843.101688:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18568) 00000100:00000001:0.0:1775776843.101825:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776843.101841:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776843.101844:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776843.101850:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.101858:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776843.101860:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.101864:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776843.101866:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776843.101868:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776843.101870:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099801472:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776843.101873:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776843.101876:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776843.101878:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.101880:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776843.101883:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776843.101885:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776843.101889:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776843.101891:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776843.101896:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776843.101901:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099801472, portal 25 00000100:00000001:0.0:1775776843.101903:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776843.101904:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257984 : -114841958293632 : ffff978d45dedf80) 00000100:00000040:0.0:1775776843.101907:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.101912:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776843.101913:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776843.101915:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776843.101917:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099801472, offset 0 00000400:00000200:0.0:1775776843.101921:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776843.101926:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.101930:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776843.101933:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776843.101938:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776843.101941:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776843.101944:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776843.101948:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4492:4492:256:4294967295] 192.168.201.139@tcp LPNI seq info [4492:4492:8:4294967295] 00000400:00000200:0.0:1775776843.101952:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.101961:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776843.101967:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776843.101972:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776843.101978:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776843.101982:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776843.102004:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776843.102008:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776843.102009:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313232119 00000100:00000010:0.0:1775776843.102013:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776843.102018:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776843.102023:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313232119 00000100:00000010:0.0:1775776843.102025:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f2373c0. 00000100:00000001:0.0:1775776843.102027:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776843.102030:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776843.102032:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776843.102035:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776843.102038:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.102042:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776843.102043:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.102045:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776843.102046:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776843.102047:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776843.102049:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801600:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776843.102051:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776843.102052:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776843.102053:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.102054:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776843.102057:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776843.102058:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776843.102060:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776843.102063:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069f8. 00000400:00000010:0.0:1775776843.102065:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8b28. 00000100:00000200:0.0:1775776843.102068:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099801600, portal 10 00000100:00000001:0.0:1775776843.102069:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776843.102070:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751264256 : -114841958287360 : ffff978d45def800) 00000100:00000040:0.0:1775776843.102074:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.102079:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776843.102080:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776843.102083:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775776843.102084:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099801600, offset 0 00000400:00000200:0.0:1775776843.102087:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776843.102089:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.102091:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776843.102093:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776843.102095:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776843.102097:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776843.102098:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776843.102100:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4493:4493:255:4294967295] 192.168.201.139@tcp LPNI seq info [4493:4493:7:4294967295] 00000400:00000200:0.0:1775776843.102103:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.102107:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776843.102110:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776843.102112:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bd00. 00000800:00000200:0.0:1775776843.102115:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776843.102118:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776843.102121:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776843.102124:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776843.102127:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776843.102129:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313230854 00000100:00000001:0.0:1775776843.102131:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313232119 00000100:00000001:0.0:1775776843.102133:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313230854 00000100:00000001:0.0:1775776843.102135:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313232119 00000100:00000001:0.0:1775776843.102137:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776843.102139:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776843.102140:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776843.102144:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.102150:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776843.102151:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.102153:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776843.102155:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776843.102168:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776843.102171:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801728:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776843.102175:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776843.102177:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776843.102178:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.102181:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776843.102184:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759cc00. 02000000:00000001:0.0:1775776843.102187:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776843.102190:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776843.102193:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306c60. 00000400:00000010:0.0:1775776843.102196:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8a18. 00000100:00000200:0.0:1775776843.102199:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099801728, portal 10 00000100:00000001:0.0:1775776843.102201:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776843.102202:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751258880 : -114841958292736 : ffff978d45dee300) 00000100:00000040:0.0:1775776843.102207:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.102214:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776843.102216:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776843.102218:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8088. 00000100:00000200:0.0:1775776843.102220:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099801728, offset 0 00000400:00000200:0.0:1775776843.102224:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776843.102228:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.102232:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776843.102234:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776843.102237:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776843.102239:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776843.102242:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776843.102244:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4494:4494:254:4294967295] 192.168.201.139@tcp LPNI seq info [4494:4494:6:4294967295] 00000400:00000200:0.0:1775776843.102248:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.102253:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776843.102258:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776843.102261:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776843.102265:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776843.102268:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776843.102271:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776843.102273:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776843.102276:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776843.102278:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313230855 00000100:00000001:0.0:1775776843.102280:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313232119 00000100:00000001:0.0:1775776843.102282:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313230855 00000100:00000001:0.0:1775776843.102284:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313232119 00000100:00000001:0.0:1775776843.102286:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776843.102288:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776843.102290:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776843.102293:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.102300:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776843.102301:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.102303:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776843.102305:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776843.102307:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776843.102310:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801856:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776843.102313:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776843.102314:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776843.102316:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.102317:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776843.102320:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775776843.102322:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776843.102325:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776843.102327:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306e70. 00000400:00000010:0.0:1775776843.102331:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8880. 00000100:00000200:0.0:1775776843.102333:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099801856, portal 4 00000100:00000001:0.0:1775776843.102335:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776843.102336:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751255296 : -114841958296320 : ffff978d45ded500) 00000100:00000040:0.0:1775776843.102341:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.102348:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776843.102350:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776843.102352:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b85d8. 00000100:00000200:0.0:1775776843.102354:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099801856, offset 0 00000400:00000200:0.0:1775776843.102358:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776843.102361:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.102364:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776843.102367:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776843.102369:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776843.102371:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776843.102374:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776843.102377:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4495:4495:253:4294967295] 192.168.201.139@tcp LPNI seq info [4495:4495:5:4294967295] 00000400:00000200:0.0:1775776843.102380:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.102386:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776843.102390:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776843.102393:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b300. 00000800:00000200:0.0:1775776843.102396:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776843.102400:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776843.102402:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776843.102404:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776843.102407:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776843.102409:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313230864 00000100:00000001:0.0:1775776843.102411:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313232119 00000100:00000001:0.0:1775776843.102413:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313230864 00000100:00000001:0.0:1775776843.102415:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313232119 00000100:00000001:0.0:1775776843.102417:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776843.102418:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776843.102420:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776843.102424:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.102429:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776843.102431:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.102432:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776843.102434:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776843.102436:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776843.102438:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801984:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776843.102441:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776843.102443:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776843.102444:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776843.102445:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776843.102448:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776843.102450:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776843.102453:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776843.102456:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513067e8. 00000400:00000010:0.0:1775776843.102459:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8f68. 00000100:00000200:0.0:1775776843.102462:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099801984, portal 4 00000100:00000001:0.0:1775776843.102463:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776843.102464:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751249920 : -114841958301696 : ffff978d45dec000) 00000100:00000040:0.0:1775776843.102469:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776843.102475:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776843.102477:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776843.102480:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8dd0. 00000100:00000200:0.0:1775776843.102482:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099801984, offset 0 00000400:00000200:0.0:1775776843.102485:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776843.102488:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.102490:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776843.102492:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776843.102493:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776843.102495:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776843.102496:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776843.102498:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4496:4496:252:4294967295] 192.168.201.139@tcp LPNI seq info [4496:4496:4:4294967295] 00000400:00000200:0.0:1775776843.102500:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776843.102503:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776843.102506:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776843.102508:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775776843.102510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776843.102512:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776843.102515:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776843.102517:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776843.102520:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776843.102521:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313230855 00000100:00000001:0.0:1775776843.102523:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313232119 00000100:00000001:0.0:1775776843.102524:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313230855 00000100:00000001:0.0:1775776843.102525:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313232119 00000100:00000001:0.0:1775776843.102526:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776843.102528:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776843.102529:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776843.102531:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776843.102536:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776843.102537:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776843.102539:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776843.102539:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776843.102541:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776843.102542:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776843.804126:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776843.804135:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776843.804137:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776843.804144:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776843.804147:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776843.804152:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776843.804154:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776843.804156:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776843.804159:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776843.867915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776843.867928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776843.867934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776843.867935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776843.867944:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776843.867945:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776843.867949:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776843.867950:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776843.868148:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776843.868152:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776843.868155:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776843.995897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776843.995905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776843.995906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776843.995911:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776843.995915:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776843.995928:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776843.995930:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776843.995933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776843.995936:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776844.379939:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313232119, at 4313232128 00000100:00000010:0.0:1775776844.379954:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775776844.380135:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313232119, at 4313232128 00000100:00000010:1.0:1775776844.380141:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f2373c0. 00000100:00000001:2.0:1775776844.827855:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776844.827861:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776844.827863:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776844.827869:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776844.827873:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776844.827876:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776844.827878:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776844.827881:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776844.827884:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776844.891924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776844.891930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776844.891936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776844.891937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776844.891940:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776844.891941:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776844.891944:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776844.891945:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776844.892072:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776844.892076:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776844.892079:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776845.019891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776845.019897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776845.019907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776845.019913:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776845.019917:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776845.019920:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776845.019922:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776845.019925:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776845.019928:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776845.851952:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776845.851969:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776845.851970:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776845.851976:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776845.851979:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776845.851983:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776845.851986:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776845.851989:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776845.851991:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776845.916072:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776845.916078:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776845.916084:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776845.916085:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776845.916089:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776845.916090:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776845.916094:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776845.916095:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776845.916153:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776845.916157:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776845.916160:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776846.043892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776846.043899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776846.043900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776846.043906:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776846.043909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776846.043913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776846.043916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776846.043918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776846.044051:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1775776846.172331:0:301876:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775776846.172376:0:301876:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775776846.172385:0:301876:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776846.172389:0:301876:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775776846.172394:0:301876:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776846.172398:0:301876:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775776846.172404:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776846.172408:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:1.0:1775776846.172414:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776846.172419:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776846.172426:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:1.0:1775776846.172428:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:1.0:1775776846.172433:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776846.172441:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776846.172515:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776846.172520:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776846.172522:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1775776846.172542:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776846.172547:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776846.172577:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776843/real 1775776846] req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776868 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776846.172589:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776846.172590:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776846.172593:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776846.172597:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776846.172599:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776846.172601:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776846.172605:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776868 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776846.172612:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776868 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776846.172618:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776846.172620:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776846.172623:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776846.172625:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776846.172628:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776868 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776846.172634:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776846.172638:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776868 ref 2 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776846.172644:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776846.172647:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776846.172649:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776846.172649:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776846.172652:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776846.172654:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776846.172658:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776846.172663:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776846.172665:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776846.172668:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776868 ref 2 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776846.172676:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099801472:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776846.172682:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776846.172684:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776846.172687:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedf80 x1862033099801472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776868 ref 2 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776846.172693:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776846.172695:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776846.172697:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776846.172698:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776846.172699:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776846.172701:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776846.173908:0:301876:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776846.173913:0:301876:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776846.173919:0:301876:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:1.0:1775776846.173926:0:301876:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776846.173930:0:301876:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776846.173932:0:301876:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:1.0:1775776846.173934:0:301876:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776846.173938:0:301876:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158c00. 02000000:00000001:1.0:1775776846.174258:0:301876:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776846.174260:0:301876:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776846.174264:0:301876:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedf80. 00000100:00000001:1.0:1775776846.174269:0:301876:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776846.174271:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776846.174273:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776846.174289:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776846.174292:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bd00. 00000400:00000200:1.0:1775776846.174298:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776846.174303:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776846.174308:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:1.0:1775776846.174310:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:1.0:1775776846.174312:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776846.174318:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776846.174345:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776846.174348:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776846.174353:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776846.174354:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776846.174359:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776846.174361:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:1.0:1775776846.174363:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776846.174366:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776846.174368:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8088 00000400:00000010:1.0:1775776846.174370:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8088. 00000100:00000001:1.0:1775776846.174371:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776846.174374:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776846.174381:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776846.174383:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776846.174388:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776846.174390:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776846.174393:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776846.174396:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b300. 00000400:00000200:1.0:1775776846.174398:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776846.174400:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776846.174402:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b85d8 00000400:00000010:1.0:1775776846.174404:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b85d8. 00000100:00000001:1.0:1775776846.174406:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776846.174410:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776846.174418:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776846.174420:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776846.174426:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776846.174428:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775776846.174431:0:301876:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775776846.174433:0:301876:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:1.0:1775776846.174435:0:301876:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775776846.174438:0:301876:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775776846.174440:0:301876:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8dd0 00000400:00000010:1.0:1775776846.174442:0:301876:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8dd0. 00000100:00000001:1.0:1775776846.174444:0:301876:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776846.174447:0:301876:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776846.174453:0:301876:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776846.174456:0:301876:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776868 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776846.174461:0:301876:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776846.174462:0:301876:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776846.174532:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776846.174535:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776846.174539:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776846.174545:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776843/real 1775776846] req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776846.174555:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776846.174557:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776846.174560:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069f8. 00000400:00000200:2.0:1775776846.174563:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8b28 00000400:00000010:2.0:1775776846.174565:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8b28. 00000100:00000001:2.0:1775776846.174567:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776846.174570:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776846.174579:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174585:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776846.174586:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174589:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776846.174590:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776846.174593:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174600:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776846.174603:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174610:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776846.174613:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174614:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776846.174615:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174617:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776846.174620:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776846.174622:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776846.174626:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776846.174628:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776846.174632:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776846.174645:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801600:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776846.174649:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776846.174651:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776843/real 1775776846] req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776846.174658:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776846.174659:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776846.174661:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306c60. 00000400:00000200:2.0:1775776846.174663:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8a18 00000400:00000010:2.0:1775776846.174664:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8a18. 00000100:00000001:2.0:1775776846.174666:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776846.174668:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776846.174675:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174681:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776846.174682:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174684:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776846.174685:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776846.174688:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174694:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776846.174697:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174703:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776846.174705:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174706:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776846.174707:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174709:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776846.174711:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776846.174713:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776846.174716:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776846.174718:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776846.174722:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776846.174729:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801728:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776846.174733:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776846.174736:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776843/real 1775776846] req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776846.174743:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776846.174744:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776846.174746:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306e70. 00000400:00000200:2.0:1775776846.174748:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8880 00000400:00000010:2.0:1775776846.174750:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8880. 00000100:00000001:2.0:1775776846.174752:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776846.174755:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776846.174761:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174768:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776846.174769:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174771:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776846.174828:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776846.174833:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174840:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776846.174844:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174851:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776846.174853:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174855:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776846.174856:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174858:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776846.174860:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776846.174863:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776846.174866:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776846.174869:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776846.174873:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776846.174881:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801856:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776846.174885:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776846.174888:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776843/real 1775776846] req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776846.174895:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776846.174897:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776846.174899:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513067e8. 00000400:00000200:2.0:1775776846.174901:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8f68 00000400:00000010:2.0:1775776846.174903:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8f68. 00000100:00000001:2.0:1775776846.174905:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776846.174908:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776846.174917:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174923:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776846.174925:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.174927:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776846.174928:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776846.174931:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.174937:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776846.175165:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.175175:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776846.175178:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.175179:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776846.175180:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.175182:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776846.175184:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776846.175186:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776846.175190:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776846.175193:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776846.175197:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776846.175204:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099801984:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776846.175208:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.175210:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776846.175212:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def800 x1862033099801600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.175218:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776846.175220:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776846.175223:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:2.0:1775776846.175227:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776846.175228:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776846.175230:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776846.175232:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776846.175234:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:2.0:1775776846.175236:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776846.175238:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776846.175239:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def800. 00000100:00000001:2.0:1775776846.175242:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776846.175243:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.175245:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776846.175248:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee300 x1862033099801728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.175254:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776846.175255:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776846.175256:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759cc00. 02000000:00000001:2.0:1775776846.175259:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776846.175260:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776846.175261:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776846.175263:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776846.175264:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:2.0:1775776846.175266:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776846.175267:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776846.175268:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee300. 00000100:00000001:2.0:1775776846.175271:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776846.175272:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.175273:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776846.175276:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded500 x1862033099801856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.175281:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776846.175283:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776846.175284:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:2.0:1775776846.175286:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776846.175287:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776846.175288:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776846.175289:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776846.175290:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158400. 02000000:00000001:2.0:1775776846.175292:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776846.175294:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776846.175295:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded500. 00000100:00000001:2.0:1775776846.175297:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776846.175298:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.175299:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776846.175303:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099801984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776868 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776846.175308:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776846.175309:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776846.175311:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:2.0:1775776846.175312:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776846.175313:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776846.175314:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776846.175316:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776846.175317:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:2.0:1775776846.175319:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776846.175320:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776846.175321:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:2.0:1775776846.175323:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776846.175324:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.175326:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.175332:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776846.175334:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776846.175336:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776846.175337:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776846.875898:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776846.875905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776846.875907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776846.875912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776846.875915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776846.875919:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.875921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776846.875924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776846.875926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776846.939912:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776846.939917:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776846.939922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776846.939925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776846.939927:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776846.939928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776846.939931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776846.939932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776846.940580:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776846.940584:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776846.940588:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776847.068870:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776847.068876:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776847.068878:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776847.068883:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776847.068886:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776847.068889:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776847.068891:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776847.068893:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776847.068895:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776847.899934:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776847.899942:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776847.899944:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776847.899949:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776847.899953:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776847.899957:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776847.899960:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776847.899963:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776847.899966:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776847.963929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776847.963936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776847.963943:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776847.963944:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776847.963947:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776847.963948:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776847.963951:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776847.963952:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776847.964246:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776847.964250:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776847.964253:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776848.091924:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776848.091932:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776848.091935:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776848.091940:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776848.091944:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776848.091947:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776848.091950:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776848.091952:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776848.091956:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776848.219903:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776848.219925:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776848.219928:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776848.219932:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776848.219937:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776848.219944:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18563 00000400:00000200:2.0:1775776848.219955:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776848.219961:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776848.219964:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776848.219966:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.219970:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776848.219972:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.219974:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776848.219978:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776848.219980:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776848.219983:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776848.219986:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776848.219988:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776848.219990:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776848.219993:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.219995:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776848.219997:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.219999:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776848.220001:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776848.220005:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220007:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776848.220010:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776848.220013:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776848.220016:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.220019:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776848.220021:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776848.220038:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c700. 00000020:00000040:2.0:1775776848.220045:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776848.220049:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776848.220050:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.220053:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776848.220056:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776848.220060:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776848.220068:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506800. 00000100:00000001:2.0:1775776848.220075:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776848.220083:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776848.220094:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776848.220099:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776848.220104:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbf2ea94] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776848.220135:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776848.220146:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776848.220149:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776848.220150:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776848.220152:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776848.220155:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776848.220157:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18563 00000400:00000200:2.0:1775776848.220161:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776848.220164:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776848.220166:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776848.220169:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220171:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776848.220173:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220175:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776848.220177:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776848.220178:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776848.220180:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776848.220182:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776848.220184:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776848.220186:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776848.220188:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220190:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776848.220192:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220193:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776848.220195:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776848.220197:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220200:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776848.220202:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776848.220207:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776848.220209:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.220211:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776848.220215:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776848.220218:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.220220:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776848.220224:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ed80. 00000020:00000040:2.0:1775776848.220226:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776848.220229:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776848.220230:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.220232:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776848.220233:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776848.220235:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776848.220239:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504c00. 00000100:00000001:2.0:1775776848.220242:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776848.220247:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776848.220254:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776848.220256:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776848.220260:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006375da9b] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776848.220267:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776848.220270:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776848.220274:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776848.220276:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776848.220277:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776848.220280:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776848.220282:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18563 00000100:00000001:0.0:1775776848.220283:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1775776848.220287:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776848.220289:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1775776848.220292:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776848.220294:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776848.220298:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776848.220299:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776848.220301:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776848.220304:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220306:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776848.220310:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:0.0:1775776848.220311:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775776848.220311:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776848.220315:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776848.220318:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776848.220320:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776848.220322:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776848.220327:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220332:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776848.220336:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776848.220337:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776848.220339:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:0.0:1775776848.220341:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776848.220342:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 02000000:00000001:0.0:1775776848.220346:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776848.220350:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1775776848.220351:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220355:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776848.220357:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776848.220360:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776848.220362:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.220364:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.220364:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776848.220366:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776848.220370:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.220372:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776848.220374:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099802112:192.168.201.139@tcp:250:kworker.0 00000100:00000010:2.0:1775776848.220377:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f800. 00000100:00000001:0.0:1775776848.220382:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1775776848.220382:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776848.220386:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776848.220388:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.220390:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775776848.220390:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776848.220393:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776848.220395:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776848.220395:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.220401:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776848.220402:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505000. 02000000:00000010:0.0:1775776848.220407:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 00000100:00000001:2.0:1775776848.220407:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.220411:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776848.220413:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776848.220420:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776848.220427:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000100:00000001:2.0:1775776848.220427:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776848.220429:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776848.220435:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8818441] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775776848.220436:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000001:2.0:1775776848.220445:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776848.220448:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1775776848.220449:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099802112, portal 25 00000100:00000001:0.0:1775776848.220454:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1775776848.220454:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776848.220457:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288712448 : -114841420839168 : ffff978d65e7c700) 00000100:00000001:2.0:1775776848.220457:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776848.220459:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776848.220463:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776848.220467:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18563 00000100:00000040:0.0:1775776848.220468:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776848.220472:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776848.220480:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1775776848.220486:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000040:0.0:1775776848.220487:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00080000:2.0:1775776848.220491:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1775776848.220493:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000040:2.0:1775776848.220495:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220498:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775776848.220499:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099802112, offset 0 00000100:00000040:2.0:1775776848.220502:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220505:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776848.220507:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776848.220508:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000400:00000200:0.0:1775776848.220511:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000020:00000001:2.0:1775776848.220512:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776848.220515:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776848.220517:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776848.220519:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776848.220523:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:0.0:1775776848.220524:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776848.220526:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776848.220530:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220532:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:0.0:1775776848.220535:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000020:00000040:2.0:1775776848.220536:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776848.220541:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000200:0.0:1775776848.220544:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775776848.220545:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776848.220547:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776848.220550:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775776848.220551:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 02000000:00000001:2.0:1775776848.220554:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776848.220558:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1775776848.220558:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776848.220562:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000400:00000200:0.0:1775776848.220564:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000008:00000001:2.0:1775776848.220567:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776848.220571:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4497:4497:256:4294967295] 192.168.201.139@tcp LPNI seq info [4497:4497:8:4294967295] 00000100:00000001:2.0:1775776848.220571:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776848.220576:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c380. 00000400:00000200:0.0:1775776848.220580:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000040:2.0:1775776848.220581:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776848.220584:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776848.220586:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.220587:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776848.220590:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776848.220592:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776848.220596:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775776848.220600:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776848.220602:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00100000:2.0:1775776848.220607:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1775776848.220613:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220620:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:0.0:1775776848.220623:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000100:00000001:2.0:1775776848.220623:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776848.220633:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:2.0:1775776848.220636:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f231afe] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1775776848.220639:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1775776848.220648:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776848.220650:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:2.0:1775776848.220651:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775776848.220655:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000100:00080000:2.0:1775776848.220656:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776848.220658:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776848.220661:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775776848.220666:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220667:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000800:00000010:0.0:1775776848.220669:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000100:00080000:2.0:1775776848.220671:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18563 00000400:00000200:0.0:1775776848.220676:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776848.220676:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.220682:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776848.220689:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000200:0.0:1775776848.220690:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000100:00080000:2.0:1775776848.220692:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1775776848.220694:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000040:2.0:1775776848.220696:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776848.220699:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775776848.220700:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776848.220704:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220707:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000200:0.0:1775776848.220708:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776848.220710:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776848.220712:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776848.220715:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776848.220719:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776848.220723:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:0.0:1775776848.220726:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1775776848.220726:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776848.220732:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000040:0.0:1775776848.220734:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776848.220735:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776848.220752:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.220753:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776848.220760:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776848.220760:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776848.220764:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:0.0:1775776848.220769:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000040:2.0:1775776848.220769:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00000001:0.0:1775776848.220773:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313237237 00000100:00080000:2.0:1775776848.220774:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000010:0.0:1775776848.220817:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776848.220824:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776848.220831:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313237238 00000100:00000010:0.0:1775776848.220835:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f2370c0. 00000100:00000001:0.0:1775776848.220837:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776848.220841:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776848.220844:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.220846:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776848.220851:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776848/real 1775776848] req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776848.220868:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776848.220869:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776848.220871:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776848.220873:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776848.220875:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776848.220877:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776848.220881:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776848.220888:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.220893:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776848.220895:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.220897:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.220898:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776848.220901:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.220907:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776848.220911:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.220916:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776848.220919:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.220921:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.220922:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.220924:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776848.220926:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776848.220928:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776848.220932:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776848.220934:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776848.220948:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776848.220957:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099802112:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776848.220962:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776848.220964:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.220970:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776848.220971:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.220973:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776848.220975:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776848.220977:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776848.220979:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802240:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776848.220982:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776848.220984:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776848.220986:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.220987:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776848.220991:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776848.220993:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776848.220996:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776848.220999:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776848.221002:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776848.221005:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099802240, portal 10 00000100:00000001:0.0:1775776848.221007:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776848.221008:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288722304 : -114841420829312 : ffff978d65e7ed80) 00000100:00000040:0.0:1775776848.221014:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221019:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776848.221021:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776848.221024:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776848.221026:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099802240, offset 0 00000400:00000200:0.0:1775776848.221029:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776848.221034:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.221046:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776848.221049:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776848.221052:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776848.221054:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776848.221057:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776848.221059:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4498:4498:256:4294967295] 192.168.201.139@tcp LPNI seq info [4498:4498:8:4294967295] 00000400:00000200:0.0:1775776848.221063:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.221069:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776848.221073:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776848.221077:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776848.221080:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776848.221083:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776848.221087:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776848.221089:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776848.221092:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776848.221094:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776848.221097:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776848.221100:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776848.221103:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776848.221105:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776848.221107:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776848.221110:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221117:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.221119:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221125:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.221127:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776848.221129:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776848.221134:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776848.221136:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776848.221139:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313235973 00000100:00000001:0.0:1775776848.221142:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313237237 00000100:00000001:0.0:1775776848.221143:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313235973 00000100:00000001:0.0:1775776848.221145:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313237238 00000100:00000001:0.0:1775776848.221147:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776848.221149:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.221151:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776848.221155:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776848/real 1775776848] req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776848.221163:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776848.221164:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776848.221166:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776848.221168:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776848.221169:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776848.221171:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776848.221174:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776848.221182:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221187:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776848.221189:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221190:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.221191:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776848.221194:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221200:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776848.221203:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221208:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776848.221211:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221212:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.221213:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221214:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776848.221216:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776848.221218:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776848.221221:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776848.221223:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776848.221226:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776848.221234:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802240:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776848.221237:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221239:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.221242:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c700 x1862033099802112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221248:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776848.221250:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776848.221252:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776848.221256:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776848.221257:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776848.221259:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776848.221261:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776848.221263:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506800. 02000000:00000001:0.0:1775776848.221266:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776848.221268:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776848.221270:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c700. 00000100:00000001:0.0:1775776848.221273:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776848.221274:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221275:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.221278:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ed80 x1862033099802240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221290:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776848.221291:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776848.221292:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776848.221294:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776848.221295:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776848.221296:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776848.221298:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776848.221299:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:0.0:1775776848.221301:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776848.221302:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776848.221303:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ed80. 00000100:00000001:0.0:1775776848.221305:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776848.221306:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221308:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221316:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776848.221318:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.221320:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776848.221322:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776848.221323:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776848.221326:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221332:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776848.221333:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.221335:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776848.221336:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776848.221338:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776848.221341:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802368:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776848.221343:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776848.221346:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776848.221348:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.221349:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776848.221352:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776848.221353:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776848.221356:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776848.221358:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776848.221360:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776848.221362:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099802368, portal 10 00000100:00000001:0.0:1775776848.221364:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776848.221365:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724992 : -114841420826624 : ffff978d65e7f800) 00000100:00000040:0.0:1775776848.221370:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221375:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776848.221377:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776848.221379:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776848.221381:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099802368, offset 0 00000400:00000200:0.0:1775776848.221385:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776848.221388:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.221392:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776848.221394:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776848.221396:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776848.221399:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776848.221401:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776848.221403:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4499:4499:256:4294967295] 192.168.201.139@tcp LPNI seq info [4499:4499:8:4294967295] 00000400:00000200:0.0:1775776848.221406:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.221411:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776848.221415:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776848.221418:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776848.221421:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776848.221424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776848.221427:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776848.221430:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776848.221433:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776848.221435:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776848.221437:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776848.221440:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776848.221442:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776848.221444:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776848.221446:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776848.221449:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221457:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.221459:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221466:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.221467:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776848.221470:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776848.221473:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776848.221476:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776848.221478:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313235974 00000100:00000001:0.0:1775776848.221480:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313237237 00000100:00000001:0.0:1775776848.221483:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313235974 00000100:00000001:0.0:1775776848.221484:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313237238 00000100:00000001:0.0:1775776848.221486:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776848.221488:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.221490:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776848.221493:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776848/real 1775776848] req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776848.221499:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776848.221501:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776848.221503:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776848.221505:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776848.221506:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776848.221508:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776848.221511:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776848.221517:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221523:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776848.221525:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221526:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.221527:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776848.221530:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221537:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776848.221541:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221546:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776848.221548:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221550:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.221551:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221552:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776848.221553:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776848.221556:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776848.221558:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776848.221561:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776848.221564:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776848.221571:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802368:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776848.221574:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776848.221577:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221583:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776848.221585:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.221587:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776848.221589:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776848.221591:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776848.221593:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802496:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776848.221596:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776848.221598:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776848.221599:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.221601:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776848.221604:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776848.221606:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776848.221609:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776848.221612:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776848.221615:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776848.221618:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099802496, portal 4 00000100:00000001:0.0:1775776848.221620:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776848.221621:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288711552 : -114841420840064 : ffff978d65e7c380) 00000100:00000040:0.0:1775776848.221626:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221632:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776848.221634:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776848.221636:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776848.221638:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099802496, offset 0 00000400:00000200:0.0:1775776848.221642:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776848.221645:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.221649:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776848.221652:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776848.221654:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776848.221656:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776848.221659:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776848.221661:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4500:4500:256:4294967295] 192.168.201.139@tcp LPNI seq info [4500:4500:8:4294967295] 00000400:00000200:0.0:1775776848.221665:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.221670:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776848.221674:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776848.221678:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776848.221681:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776848.221683:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776848.221687:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776848.221689:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776848.221703:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776848.221705:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776848.221707:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776848.221710:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776848.221712:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776848.221713:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776848.221715:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776848.221717:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221722:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.221723:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221727:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.221728:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776848.221730:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776848.221732:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776848.221734:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776848.221735:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313235983 00000100:00000001:0.0:1775776848.221755:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313237237 00000100:00000001:0.0:1775776848.221758:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313235983 00000100:00000001:0.0:1775776848.221760:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313237238 00000100:00000001:0.0:1775776848.221762:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776848.221764:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.221765:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776848.221769:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776848/real 1775776848] req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776848.221777:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776848.221805:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776848.221807:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776848.221810:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776848.221811:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776848.221813:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776848.221816:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776848.221823:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221828:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776848.221830:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221832:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.221833:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776848.221836:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221841:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776848.221844:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221851:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776848.221853:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221855:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.221856:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221857:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776848.221859:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776848.221862:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776848.221865:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776848.221867:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776848.221870:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776848.221877:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802496:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776848.221880:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221881:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.221884:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f800 x1862033099802368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221891:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776848.221892:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776848.221894:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776848.221896:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776848.221898:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776848.221899:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776848.221901:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776848.221902:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:0.0:1775776848.221906:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776848.221907:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776848.221908:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f800. 00000100:00000001:0.0:1775776848.221910:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776848.221911:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221912:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.221915:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099802496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.221921:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776848.221922:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776848.221923:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776848.221926:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776848.221927:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776848.221928:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776848.221930:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776848.221931:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:0.0:1775776848.221933:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776848.221934:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776848.221936:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:0.0:1775776848.221938:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776848.221939:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221940:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.221944:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776848.221946:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.221947:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776848.221949:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.222516:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776848.222522:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776848.222527:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776848.222530:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.222532:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776848.222535:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776848.222537:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.222539:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776848.222546:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7df80. 00000020:00000040:2.0:1775776848.222550:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776848.222553:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776848.222554:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.222557:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776848.222559:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776848.222562:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776848.222567:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:2.0:1775776848.222572:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776848.222579:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776848.222587:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776848.222591:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776848.222595:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009deec8f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776848.222617:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776848.222625:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18573) 00000100:00000001:0.0:1775776848.223023:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776848.223026:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776848.223028:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776848.223034:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.223278:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776848.223280:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.223283:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776848.223285:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776848.223287:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776848.223291:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802624:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776848.223294:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776848.223297:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776848.223299:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776848.223302:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776848.223308:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776848.223310:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776848.223316:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776848.223320:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776848.223324:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776848.223329:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099802624, portal 4 00000100:00000001:0.0:1775776848.223331:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776848.223332:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288718720 : -114841420832896 : ffff978d65e7df80) 00000100:00000040:0.0:1775776848.223338:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.223344:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776848.223347:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776848.223348:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776848.223351:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099802624, offset 0 00000400:00000200:0.0:1775776848.223356:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776848.223362:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.223368:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776848.223372:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776848.223375:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776848.223378:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776848.223380:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776848.223383:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4501:4501:256:4294967295] 192.168.201.139@tcp LPNI seq info [4501:4501:8:4294967295] 00000400:00000200:0.0:1775776848.223387:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776848.223395:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776848.223399:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776848.223402:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776848.223407:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776848.223410:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776848.223415:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776848.223417:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776848.223420:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776848.223422:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776848.223427:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776848.223430:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776848.223434:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776848.223436:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776848.223439:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776848.223442:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.223450:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.223452:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776873 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.223458:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.223459:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776848.223470:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776848.223474:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776848.223477:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776848.223479:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313235976 00000100:00000001:0.0:1775776848.223482:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313237237 00000100:00000001:0.0:1775776848.223484:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313235976 00000100:00000001:0.0:1775776848.223486:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313237238 00000100:00000001:0.0:1775776848.223488:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776848.223491:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.223493:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776848.223497:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776848/real 1775776848] req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776848.223504:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776848.223505:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776848.223507:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776848.223509:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776848.223511:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776848.223512:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776848.223515:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776848.223522:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.223529:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776848.223530:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.223533:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.223534:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776848.223537:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.223543:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776848.223546:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.223552:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776848.223554:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.223555:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776848.223557:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.223559:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776848.223561:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776848.223563:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776848.223566:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776848.223569:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776848.223572:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776848.223579:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802624:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776848.223584:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.223585:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776848.223588:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7df80 x1862033099802624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776873 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776848.223672:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776848.223674:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776848.223677:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776848.223681:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776848.223683:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776848.223684:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776848.223686:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776848.223688:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:0.0:1775776848.223692:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776848.223693:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776848.223695:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7df80. 00000100:00000001:0.0:1775776848.223698:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776848.223699:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.223702:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776848.223707:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776848.223709:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776848.223711:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776848.223712:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776848.923882:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776848.923890:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776848.923893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776848.923898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776848.923911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776848.923915:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776848.923917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776848.923920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776848.923922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776848.987946:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776848.987954:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776848.987957:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776848.987960:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776848.987964:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776848.987967:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776848.987969:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776848.987971:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776848.987975:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776848.987977:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776848.987980:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776848.987982:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776848.987985:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776848.987986:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776848.988592:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776848.988596:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776848.988599:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776849.115894:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776849.115901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776849.115903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776849.115911:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776849.115914:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776849.115918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776849.115921:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776849.115923:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776849.115926:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776849.499891:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313237237, at 4313237248 00000100:00000010:0.0:1775776849.499907:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775776849.500079:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313237238, at 4313237248 00000100:00000010:1.0:1775776849.500085:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f2370c0. 00000100:00000001:2.0:1775776849.947920:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776849.947928:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776849.947930:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776849.947936:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776849.947940:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776849.947944:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776849.947946:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776849.947949:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776849.947952:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776850.011932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776850.011937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776850.011943:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776850.011945:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776850.011948:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776850.011949:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776850.011952:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776850.011953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776850.012379:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776850.012383:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776850.012387:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776850.142229:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776850.142236:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776850.142238:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776850.142243:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776850.142247:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776850.142251:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776850.142254:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776850.142256:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776850.142259:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776850.971923:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776850.971931:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776850.971933:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776850.971939:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776850.971943:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776850.971946:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776850.971949:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776850.971951:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776850.971954:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035898:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776851.035905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776851.035914:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776851.035916:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776851.035920:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035922:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776851.035925:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035927:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035929:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776851.035944:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776851.035945:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776851.035947:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035948:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776851.035949:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035951:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035953:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776851.035958:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776851.035959:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776851.035960:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035962:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776851.035963:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035964:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776851.035988:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776851.036090:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776851.036092:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776851.036094:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776851.163890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776851.163896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776851.163898:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776851.163903:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776851.163907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776851.163910:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776851.163912:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776851.163915:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776851.163917:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776851.995901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776851.995909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776851.995911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776851.995917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776851.995921:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776851.995925:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776851.995927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776851.995929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776851.995932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776852.060442:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776852.060449:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776852.060454:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776852.060456:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776852.060459:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776852.060460:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776852.060463:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776852.060464:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776852.060536:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776852.060540:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776852.060543:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776852.187860:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776852.187867:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776852.187869:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776852.187874:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776852.187878:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776852.187881:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776852.187883:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776852.187886:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776852.187888:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.019902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776853.019908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776853.019910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.019916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776853.019919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776853.019923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.019925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776853.019928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.019930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776853.084461:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776853.084468:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776853.084475:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776853.084476:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776853.084479:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776853.084480:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776853.084483:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776853.084484:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776853.086008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776853.086014:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776853.086019:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776853.212132:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776853.212142:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776853.212144:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776853.212150:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776853.212154:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776853.212159:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776853.212161:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776853.212164:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776853.212167:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776853.341838:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776853.341848:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776853.341851:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776853.341855:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776853.341859:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776853.341866:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18568 00000400:00000200:2.0:1775776853.341875:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.341880:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776853.341883:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776853.341886:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.341889:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.341892:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.341894:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776853.341898:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776853.341900:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776853.341902:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776853.341905:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776853.341907:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776853.341909:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776853.341913:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.341915:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.341917:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.341919:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776853.341922:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776853.341925:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776853.341927:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.341930:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776853.341933:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.341936:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.341938:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776853.341940:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776853.341946:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7df80. 00000020:00000040:2.0:1775776853.341950:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776853.341954:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776853.341956:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.341959:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776853.341962:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776853.341967:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776853.341975:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:2.0:1775776853.341983:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.341993:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776853.342005:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776853.342010:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.342015:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009deec8f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342036:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776853.342039:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776853.342042:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776853.342043:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776853.342044:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776853.342047:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776853.342049:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18568 00000400:00000200:2.0:1775776853.342053:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.342057:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776853.342059:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776853.342062:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342064:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.342065:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342067:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776853.342069:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776853.342071:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776853.342072:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776853.342074:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776853.342075:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776853.342077:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776853.342079:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342081:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.342082:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342084:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776853.342086:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776853.342088:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342090:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.342092:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776853.342097:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.342099:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342101:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776853.342105:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776853.342107:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342109:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776853.342113:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c380. 00000020:00000040:2.0:1775776853.342115:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776853.342117:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776853.342118:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342120:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776853.342121:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776853.342123:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776853.342127:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506c00. 00000100:00000001:2.0:1775776853.342130:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.342135:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776853.342142:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776853.342145:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.342148:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f231afe] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342155:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776853.342158:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776853.342161:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776853.342162:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776853.342164:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776853.342166:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776853.342168:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18568 00000400:00000200:2.0:1775776853.342171:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.342174:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776853.342176:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776853.342178:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342180:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.342181:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342183:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776853.342185:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776853.342186:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776853.342187:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776853.342189:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776853.342191:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776853.342193:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776853.342196:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342197:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.342199:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342200:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776853.342202:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776853.342205:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342207:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.342208:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776853.342211:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.342212:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342214:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776853.342216:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776853.342218:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342220:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776853.342228:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f800. 00000020:00000040:2.0:1775776853.342231:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776853.342233:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776853.342234:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342236:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776853.342238:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776853.342239:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776853.342243:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506000. 00000100:00000001:2.0:1775776853.342245:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.342249:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776853.342254:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776853.342256:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.342260:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8818441] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342267:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776853.342269:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776853.342272:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776853.342273:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776853.342275:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776853.342277:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776853.342279:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18568 00000400:00000200:2.0:1775776853.342283:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.342285:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776853.342286:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776853.342289:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342290:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.342292:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342294:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776853.342295:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776853.342297:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776853.342298:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776853.342300:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776853.342302:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776853.342303:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776853.342306:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342308:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.342309:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342311:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776853.342313:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776853.342316:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342318:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.342320:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776853.342323:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.342324:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342326:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776853.342329:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776853.342331:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342332:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776853.342336:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ed80. 00000020:00000040:2.0:1775776853.342338:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776853.342340:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776853.342341:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342343:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776853.342344:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776853.342346:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776853.342349:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775776853.342352:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.342356:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776853.342363:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776853.342365:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.342367:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006375da9b] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342373:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776853.342376:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776853.342379:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776853.342380:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776853.342381:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776853.342383:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776853.342386:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18568 00000400:00000200:2.0:1775776853.342390:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.342393:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776853.342395:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776853.342398:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342399:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.342402:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342404:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776853.342406:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776853.342407:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776853.342408:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776853.342410:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776853.342412:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776853.342414:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776853.342416:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342418:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776853.342419:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342421:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776853.342423:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776853.342425:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776853.342427:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.342429:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776853.342432:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.342433:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342435:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776853.342437:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776853.342439:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342440:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776853.342443:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c700. 00000020:00000040:2.0:1775776853.342445:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776853.342447:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776853.342448:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342449:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776853.342450:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776853.342452:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776853.342455:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:2.0:1775776853.342458:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.342463:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776853.342469:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776853.342471:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.342475:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbf2ea94] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342482:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.342487:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18578) 00000100:00000001:2.0:1775776853.342514:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776853.342518:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776853.342521:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776853.342525:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342531:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776853.342533:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.342536:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776853.342538:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.342540:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.342544:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099802752:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776853.342547:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776853.342551:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776853.342553:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.342556:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776853.342559:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776853.342561:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776853.342566:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776853.342569:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776853.342575:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776853.342582:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099802752, portal 25 00000100:00000001:2.0:1775776853.342584:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776853.342585:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288718720 : -114841420832896 : ffff978d65e7df80) 00000100:00000040:2.0:1775776853.342590:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342596:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776853.342598:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776853.342601:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776853.342603:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099802752, offset 0 00000400:00000200:2.0:1775776853.342609:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776853.342614:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.342619:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776853.342623:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776853.342627:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776853.342630:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776853.342633:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776853.342637:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4502:4502:256:4294967295] 192.168.201.139@tcp LPNI seq info [4502:4502:8:4294967295] 00000400:00000200:2.0:1775776853.342641:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.342661:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776853.342668:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.342673:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776853.342680:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776853.342683:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776853.342688:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776853.342691:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776853.342695:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.342697:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776853.342701:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776853.342706:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776853.342711:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776853.342713:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776853.342718:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776853.342722:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342730:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.342733:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342740:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342741:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776853.342747:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776853.342751:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776853.342753:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313242359 00000100:00000010:2.0:1775776853.342757:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b5a0. 00000100:00000001:2.0:1775776853.342763:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776853.342769:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313242359 00000100:00000010:2.0:1775776853.342772:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b7e0. 00000100:00000001:2.0:1775776853.342774:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776853.342777:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776853.342798:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.342801:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776853.342806:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776853/real 1775776853] req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776853.342817:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776853.342819:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776853.342820:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776853.342823:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776853.342825:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776853.342828:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776853.342832:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776853.342840:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342846:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776853.342848:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.342850:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.342853:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776853.342856:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342864:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776853.342868:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342875:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776853.342878:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.342880:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.342881:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.342884:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776853.342886:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776853.342890:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776853.342894:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776853.342897:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776853.342902:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776853.342911:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099802752:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776853.342917:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776853.342920:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342926:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776853.342928:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.342930:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776853.342932:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.342934:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.342936:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802880:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776853.342939:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776853.342942:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776853.342943:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.342945:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776853.342949:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776853.342950:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776853.342954:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776853.342956:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776853.342959:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776853.342962:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099802880, portal 10 00000100:00000001:2.0:1775776853.342963:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776853.342965:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288711552 : -114841420840064 : ffff978d65e7c380) 00000100:00000040:2.0:1775776853.342972:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.342977:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776853.342979:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776853.342981:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776853.342984:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099802880, offset 0 00000400:00000200:2.0:1775776853.342987:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776853.342992:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.342996:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776853.342999:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776853.343001:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776853.343004:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776853.343006:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776853.343010:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4503:4503:256:4294967295] 192.168.201.139@tcp LPNI seq info [4503:4503:8:4294967295] 00000400:00000200:2.0:1775776853.343014:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.343020:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776853.343025:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.343028:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776853.343032:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776853.343034:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776853.343038:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776853.343041:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776853.343044:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.343045:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776853.343048:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776853.343051:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776853.343054:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776853.343056:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776853.343058:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776853.343061:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343069:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.343071:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343077:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.343078:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776853.343081:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.343085:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776853.343088:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776853.343090:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313241095 00000100:00000001:2.0:1775776853.343093:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313242359 00000100:00000001:2.0:1775776853.343094:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313241095 00000100:00000001:2.0:1775776853.343096:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313242359 00000100:00000001:2.0:1775776853.343098:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776853.343100:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.343102:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776853.343105:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776853/real 1775776853] req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776853.343113:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776853.343114:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776853.343116:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776853.343118:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776853.343119:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776853.343121:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776853.343125:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776853.343133:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343139:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776853.343140:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343142:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.343143:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776853.343147:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343153:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776853.343157:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343164:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776853.343166:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343168:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.343170:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343172:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776853.343174:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776853.343177:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776853.343181:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776853.343184:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776853.343189:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776853.343197:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099802880:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776853.343200:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776853.343203:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343215:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776853.343216:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.343218:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776853.343220:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.343222:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.343225:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803008:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776853.343229:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776853.343231:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776853.343232:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.343234:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776853.343237:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776853.343239:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776853.343242:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776853.343245:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776853.343248:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776853.343250:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099803008, portal 10 00000100:00000001:2.0:1775776853.343252:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776853.343253:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724992 : -114841420826624 : ffff978d65e7f800) 00000100:00000040:2.0:1775776853.343259:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343264:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776853.343266:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776853.343268:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776853.343270:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099803008, offset 0 00000400:00000200:2.0:1775776853.343274:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776853.343278:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.343281:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776853.343283:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776853.343285:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776853.343288:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776853.343290:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776853.343293:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4504:4504:256:4294967295] 192.168.201.139@tcp LPNI seq info [4504:4504:8:4294967295] 00000400:00000200:2.0:1775776853.343297:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.343303:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776853.343307:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.343311:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776853.343314:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776853.343317:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776853.343321:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776853.343323:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776853.343327:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.343329:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776853.343331:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776853.343334:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776853.343337:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776853.343338:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776853.343341:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776853.343343:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343351:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.343353:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343359:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.343361:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776853.343363:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.343366:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776853.343369:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776853.343371:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313241096 00000100:00000001:2.0:1775776853.343374:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313242359 00000100:00000001:2.0:1775776853.343376:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313241096 00000100:00000001:2.0:1775776853.343378:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313242359 00000100:00000001:2.0:1775776853.343380:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776853.343383:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.343384:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776853.343388:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776853/real 1775776853] req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776853.343394:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776853.343396:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776853.343397:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776853.343399:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776853.343401:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776853.343403:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776853.343405:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776853.343411:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343417:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776853.343418:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343420:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.343421:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776853.343424:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343430:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776853.343433:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343440:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776853.343442:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343443:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.343445:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343446:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776853.343447:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776853.343450:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776853.343453:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776853.343455:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776853.343458:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776853.343464:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803008:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776853.343468:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776853.343471:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343478:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776853.343480:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.343482:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776853.343483:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.343485:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.343487:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803136:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776853.343490:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776853.343492:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776853.343493:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.343495:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776853.343497:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776853.343499:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776853.343503:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776853.343505:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776853.343508:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776853.343511:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099803136, portal 4 00000100:00000001:2.0:1775776853.343513:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776853.343514:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288722304 : -114841420829312 : ffff978d65e7ed80) 00000100:00000040:2.0:1775776853.343519:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343525:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776853.343528:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776853.343530:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776853.343533:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099803136, offset 0 00000400:00000200:2.0:1775776853.343536:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776853.343540:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.343543:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776853.343546:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776853.343548:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776853.343550:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776853.343552:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776853.343555:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4505:4505:256:4294967295] 192.168.201.139@tcp LPNI seq info [4505:4505:8:4294967295] 00000400:00000200:2.0:1775776853.343559:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.343564:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776853.343568:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.343571:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776853.343574:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776853.343577:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776853.343581:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776853.343583:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776853.343587:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.343589:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776853.343591:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776853.343594:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776853.343596:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776853.343598:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776853.343600:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776853.343603:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343610:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.343613:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343619:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.343620:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776853.343624:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.343626:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776853.343629:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776853.343631:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313241105 00000100:00000001:2.0:1775776853.343633:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313242359 00000100:00000001:2.0:1775776853.343635:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313241105 00000100:00000001:2.0:1775776853.343637:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313242359 00000100:00000001:2.0:1775776853.343639:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776853.343642:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.343643:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776853.343658:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776853/real 1775776853] req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776853.343668:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776853.343669:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776853.343671:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776853.343673:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776853.343675:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776853.343677:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776853.343680:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776853.343687:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343694:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776853.343695:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343697:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.343698:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776853.343701:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343707:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776853.343711:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343718:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776853.343720:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343722:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.343723:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343725:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776853.343727:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776853.343729:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776853.343732:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776853.343734:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776853.343738:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776853.343745:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803136:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776853.343748:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776853.343751:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343757:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776853.343759:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.343760:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776853.343762:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776853.343764:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776853.343767:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803264:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776853.343770:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776853.343771:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776853.343773:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776853.343774:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776853.343777:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776853.343806:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776853.343809:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776853.343812:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776853.343815:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776853.343818:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099803264, portal 4 00000100:00000001:2.0:1775776853.343819:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776853.343821:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288712448 : -114841420839168 : ffff978d65e7c700) 00000100:00000040:2.0:1775776853.343826:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343832:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776853.343834:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776853.343836:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776853.343838:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099803264, offset 0 00000400:00000200:2.0:1775776853.343842:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776853.343846:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.343849:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776853.343852:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776853.343854:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776853.343858:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776853.343861:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776853.343865:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4506:4506:256:4294967295] 192.168.201.139@tcp LPNI seq info [4506:4506:8:4294967295] 00000400:00000200:2.0:1775776853.343869:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776853.343876:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776853.343880:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.343883:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776853.343887:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776853.343889:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776853.343893:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776853.343896:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776853.343899:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776853.343901:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776853.343903:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776853.343906:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776853.343909:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776853.343910:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776853.343912:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776853.343915:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343922:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.343924:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776878 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343929:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.343930:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776853.343933:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776853.343935:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776853.343938:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776853.343940:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313241097 00000100:00000001:2.0:1775776853.343942:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313242359 00000100:00000001:2.0:1775776853.343944:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313241097 00000100:00000001:2.0:1775776853.343946:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313242359 00000100:00000001:2.0:1775776853.343947:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776853.343949:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.343951:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776853.343954:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776853/real 1775776853] req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776853.343960:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776853.343961:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776853.343963:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776853.343965:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776853.343966:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776853.343968:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776853.343970:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776853.343977:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.343987:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776853.343988:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.343990:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.343991:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776853.343994:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.344000:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776853.344004:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.344010:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776853.344012:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344014:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776853.344015:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344017:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776853.344018:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776853.344021:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776853.344023:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776853.344025:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776853.344029:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776853.344035:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803264:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776853.344039:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344040:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.344043:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7df80 x1862033099802752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.344049:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776853.344051:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776853.344053:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776853.344056:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776853.344058:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776853.344060:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776853.344062:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776853.344064:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776853.344066:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776853.344068:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776853.344070:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7df80. 00000100:00000001:2.0:1775776853.344072:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776853.344073:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344075:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.344078:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099802880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.344083:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776853.344084:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776853.344087:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776853.344088:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776853.344089:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776853.344090:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776853.344093:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776853.344094:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775776853.344096:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776853.344097:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776853.344098:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:2.0:1775776853.344100:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776853.344101:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344102:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.344104:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f800 x1862033099803008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.344109:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776853.344110:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776853.344111:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776853.344113:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776853.344114:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776853.344115:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776853.344117:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776853.344119:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506000. 02000000:00000001:2.0:1775776853.344121:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776853.344122:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776853.344123:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f800. 00000100:00000001:2.0:1775776853.344125:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776853.344126:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344127:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.344130:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ed80 x1862033099803136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.344135:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776853.344136:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776853.344138:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776853.344139:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776853.344140:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776853.344141:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776853.344143:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776853.344144:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776853.344146:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776853.344147:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776853.344148:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ed80. 00000100:00000001:2.0:1775776853.344149:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776853.344150:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344152:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776853.344154:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c700 x1862033099803264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776878 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776853.344159:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776853.344160:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776853.344161:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776853.344163:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776853.344164:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776853.344166:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776853.344167:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776853.344168:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776853.344171:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776853.344172:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776853.344173:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c700. 00000100:00000001:2.0:1775776853.344175:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776853.344176:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344178:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776853.344185:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776853.344187:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776853.344190:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776853.344191:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776854.043904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776854.043911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776854.043913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776854.043918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776854.043922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776854.043927:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776854.043929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776854.043931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776854.043934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776854.108768:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776854.108775:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776854.108796:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776854.108797:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776854.108801:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776854.108803:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776854.108806:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776854.108807:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776854.109103:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776854.109106:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776854.109109:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776854.235891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776854.235898:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776854.235900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776854.235905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776854.235909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776854.235913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776854.235915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776854.235917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776854.235920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776854.620204:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313242359, at 4313242368 00000100:00000010:0.0:1775776854.620219:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000040:1.0:1775776854.620703:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313242359, at 4313242368 00000100:00000010:1.0:1775776854.620710:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:2.0:1775776855.067898:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776855.067905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776855.067907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776855.067913:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776855.067917:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776855.067921:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776855.067923:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776855.067925:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776855.067927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776855.131952:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776855.131958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776855.131964:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776855.131966:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776855.131969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776855.131970:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776855.131974:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776855.131975:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776855.132851:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776855.132855:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776855.132858:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776855.259921:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776855.259929:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776855.259931:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776855.259936:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776855.259940:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776855.259945:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776855.259948:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776855.259950:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776855.259953:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776856.092354:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776856.092361:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776856.092363:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776856.092368:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776856.092372:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776856.092375:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776856.092378:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776856.092380:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776856.092382:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776856.156121:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776856.156127:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776856.156133:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776856.156134:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776856.156138:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776856.156139:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776856.156143:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776856.156144:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776856.156269:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776856.156273:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776856.156281:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776856.284056:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776856.284063:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776856.284065:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776856.284071:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776856.284074:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776856.284078:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776856.284080:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776856.284083:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776856.284085:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776857.116624:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776857.116631:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776857.116633:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776857.116639:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776857.116643:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776857.116647:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776857.116650:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776857.116652:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776857.116655:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776857.179914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776857.179920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776857.179926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776857.179928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776857.179931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776857.179932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776857.179935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776857.179937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776857.181660:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776857.181666:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776857.181670:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776857.307926:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776857.307932:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776857.307934:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776857.307939:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776857.307942:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776857.307946:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776857.307948:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776857.307951:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776857.307953:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.140755:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776858.140763:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776858.140764:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776858.140769:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776858.140773:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776858.140777:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.140818:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776858.140820:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.140824:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776858.204015:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776858.204021:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776858.204028:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776858.204029:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776858.204032:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776858.204035:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776858.204038:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776858.204039:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776858.204967:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776858.204972:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776858.204975:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776858.331883:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776858.331890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776858.331892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776858.331906:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776858.331909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776858.331913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776858.331916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776858.331918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776858.331921:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776858.464540:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776858.464550:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776858.464553:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776858.464557:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776858.464571:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776858.464578:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18573 00000400:00000200:0.0:1775776858.464587:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776858.464593:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776858.464596:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776858.464600:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464603:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.464605:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464608:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776858.464611:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776858.464614:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776858.464616:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776858.464619:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776858.464621:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776858.464623:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776858.464626:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464628:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.464629:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464631:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776858.464634:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776858.464637:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464639:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776858.464642:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776858.464644:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776858.464647:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.464650:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776858.464652:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776858.464661:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776858.464664:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776858.464668:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776858.464670:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.464672:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776858.464675:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776858.464679:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776858.464686:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759cc00. 00000100:00000001:0.0:1775776858.464692:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776858.464699:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776858.464709:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776858.464714:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776858.464718:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776858.464747:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776858.464751:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776858.464754:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776858.464755:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776858.464756:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776858.464759:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776858.464761:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18573 00000400:00000200:0.0:1775776858.464764:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776858.464767:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776858.464769:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776858.464772:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464774:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.464776:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464800:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776858.464802:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776858.464803:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776858.464805:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776858.464807:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776858.464809:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776858.464811:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776858.464814:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464816:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.464818:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464819:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776858.464821:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776858.464824:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464826:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776858.464828:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776858.464833:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776858.464835:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.464837:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776858.464841:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776858.464843:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.464845:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776858.464849:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776858.464851:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776858.464854:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776858.464855:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.464856:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776858.464858:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776858.464859:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776858.464862:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775776858.464865:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776858.464870:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776858.464877:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776858.464878:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776858.464882:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776858.464888:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776858.464890:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776858.464904:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776858.464906:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776858.464907:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776858.464909:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776858.464912:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18573 00000400:00000200:0.0:1775776858.464916:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776858.464919:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776858.464921:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776858.464924:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464925:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.464927:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464929:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776858.464931:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776858.464932:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776858.464933:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776858.464935:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776858.464937:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776858.464938:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776858.464941:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464943:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.464944:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464946:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776858.464948:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776858.464951:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776858.464953:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776858.464954:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776858.464957:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776858.464958:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.464960:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776858.464962:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776858.464964:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.464966:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776858.464969:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776858.464971:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776858.464974:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776858.464975:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.464977:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776858.464978:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776858.464980:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776858.464983:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776858.464986:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776858.464990:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776858.464998:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776858.465005:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776858.465009:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776858.465016:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776858.465018:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776858.465021:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776858.465022:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776858.465023:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776858.465025:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776858.465028:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18573 00000400:00000200:0.0:1775776858.465031:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776858.465034:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776858.465035:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776858.465038:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465039:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.465041:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465043:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776858.465045:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776858.465046:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776858.465048:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776858.465049:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776858.465051:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776858.465053:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776858.465056:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465058:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.465060:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465062:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776858.465064:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776858.465067:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465070:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776858.465071:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776858.465074:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776858.465076:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.465078:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776858.465080:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776858.465082:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.465083:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776858.465087:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776858.465089:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776858.465091:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776858.465093:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.465101:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776858.465103:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776858.465105:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776858.465108:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776858.465112:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776858.465116:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776858.465123:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776858.465125:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776858.465128:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776858.465135:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776858.465137:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776858.465140:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776858.465142:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776858.465144:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776858.465146:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776858.465148:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18573 00000400:00000200:0.0:1775776858.465152:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776858.465155:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776858.465157:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776858.465159:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465161:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.465163:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465165:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776858.465167:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776858.465168:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776858.465169:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776858.465171:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776858.465173:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776858.465175:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776858.465178:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465180:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776858.465181:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465183:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776858.465185:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776858.465187:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776858.465189:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776858.465190:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776858.465193:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776858.465195:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.465197:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776858.465199:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776858.465201:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.465203:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776858.465206:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776858.465208:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776858.465210:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776858.465211:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776858.465213:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776858.465214:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776858.465216:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776858.465219:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776858.465222:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776858.465344:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776858.465352:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776858.465354:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776858.465357:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776858.465365:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776858.465370:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18583) 00000100:00000001:2.0:1775776858.465430:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776858.465434:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776858.465436:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776858.465442:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465449:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776858.465451:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.465454:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776858.465457:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776858.465459:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776858.465463:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099803392:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776858.465467:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776858.465470:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776858.465472:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.465474:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776858.465478:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776858.465480:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776858.465485:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776858.465489:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776858.465495:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776858.465502:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099803392, portal 25 00000100:00000001:2.0:1775776858.465504:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776858.465506:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741753472 : -114841967798144 : ffff978d454dd880) 00000100:00000040:2.0:1775776858.465511:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465519:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776858.465521:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776858.465523:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776858.465527:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099803392, offset 0 00000400:00000200:2.0:1775776858.465533:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776858.465540:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.465545:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776858.465549:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776858.465553:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776858.465557:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776858.465570:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776858.465574:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4507:4507:256:4294967295] 192.168.201.139@tcp LPNI seq info [4507:4507:8:4294967295] 00000400:00000200:2.0:1775776858.465579:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.465588:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776858.465593:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.465598:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776858.465604:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776858.465608:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776858.465613:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776858.465616:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776858.465620:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.465623:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776858.465628:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776858.465633:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776858.465637:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776858.465639:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776858.465643:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776858.465647:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465656:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.465659:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465666:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.465667:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776858.465672:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776858.465675:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776858.465678:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313247482 00000100:00000010:2.0:1775776858.465683:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775776858.465689:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776858.465695:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313247482 00000100:00000010:2.0:1775776858.465697:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bba0. 00000100:00000001:2.0:1775776858.465700:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776858.465703:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776858.465706:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.465708:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776858.465711:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776858/real 1775776858] req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776858.465721:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776858.465723:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776858.465725:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776858.465727:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776858.465729:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776858.465731:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776858.465734:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776858.465743:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465753:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776858.465755:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.465757:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.465759:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776858.465762:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465768:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776858.465772:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465797:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776858.465799:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.465801:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.465802:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.465804:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776858.465806:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776858.465809:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776858.465813:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776858.465815:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776858.465819:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776858.465828:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099803392:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776858.465832:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776858.465835:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465841:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776858.465842:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.465844:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776858.465846:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776858.465848:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776858.465851:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803520:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776858.465854:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776858.465856:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776858.465857:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.465859:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776858.465862:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776858.465863:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776858.465872:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776858.465874:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776858.465877:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776858.465880:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099803520, portal 10 00000100:00000001:2.0:1775776858.465882:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776858.465883:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741760640 : -114841967790976 : ffff978d454df480) 00000100:00000040:2.0:1775776858.465889:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.465904:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776858.465907:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776858.465909:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776858.465912:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099803520, offset 0 00000400:00000200:2.0:1775776858.465916:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776858.465920:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.465923:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776858.465926:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776858.465928:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776858.465931:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776858.465933:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776858.465935:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4508:4508:256:4294967295] 192.168.201.139@tcp LPNI seq info [4508:4508:8:4294967295] 00000400:00000200:2.0:1775776858.465939:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.465945:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776858.465949:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.465952:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776858.466054:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776858.466057:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776858.466062:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776858.466064:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776858.466067:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.466069:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776858.466073:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776858.466076:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776858.466080:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776858.466082:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776858.466084:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776858.466088:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466096:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.466098:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466105:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.466106:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776858.466109:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776858.466113:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776858.466116:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776858.466118:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313246218 00000100:00000001:2.0:1775776858.466121:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313247482 00000100:00000001:2.0:1775776858.466123:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313246218 00000100:00000001:2.0:1775776858.466124:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313247482 00000100:00000001:2.0:1775776858.466126:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776858.466128:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.466130:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776858.466133:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776858/real 1775776858] req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776858.466145:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776858.466146:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776858.466148:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776858.466150:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776858.466151:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776858.466153:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776858.466156:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776858.466164:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466170:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776858.466171:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466173:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.466174:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776858.466177:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466183:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776858.466187:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466193:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776858.466195:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466197:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.466198:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466200:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776858.466202:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776858.466204:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776858.466207:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776858.466210:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776858.466213:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776858.466221:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803520:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776858.466225:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776858.466238:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466245:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776858.466246:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.466248:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776858.466250:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776858.466252:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776858.466254:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803648:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776858.466257:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776858.466259:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776858.466261:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.466262:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776858.466266:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776858.466267:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776858.466270:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776858.466272:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776858.466275:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776858.466277:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099803648, portal 10 00000100:00000001:2.0:1775776858.466279:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776858.466281:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741758848 : -114841967792768 : ffff978d454ded80) 00000100:00000040:2.0:1775776858.466286:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466292:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776858.466295:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776858.466297:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776858.466299:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099803648, offset 0 00000400:00000200:2.0:1775776858.466303:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776858.466306:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.466310:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776858.466312:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776858.466315:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776858.466317:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776858.466319:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776858.466322:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4509:4509:256:4294967295] 192.168.201.139@tcp LPNI seq info [4509:4509:8:4294967295] 00000400:00000200:2.0:1775776858.466326:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.466332:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776858.466336:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.466339:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776858.466343:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776858.466345:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776858.466349:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776858.466351:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776858.466355:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.466356:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776858.466359:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776858.466362:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776858.466365:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776858.466366:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776858.466368:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776858.466371:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466378:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.466381:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466387:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.466389:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776858.466391:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776858.466394:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776858.466397:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776858.466399:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313246219 00000100:00000001:2.0:1775776858.466401:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313247482 00000100:00000001:2.0:1775776858.466403:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313246219 00000100:00000001:2.0:1775776858.466405:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313247482 00000100:00000001:2.0:1775776858.466406:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776858.466408:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.466410:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776858.466413:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776858/real 1775776858] req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776858.466421:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776858.466422:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776858.466423:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776858.466425:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776858.466427:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776858.466429:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776858.466431:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776858.466438:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466444:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776858.466446:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466447:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.466448:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776858.466451:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466457:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776858.466461:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466467:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776858.466469:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466470:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.466472:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466473:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776858.466475:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776858.466477:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776858.466480:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776858.466481:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776858.466485:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776858.466492:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803648:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776858.466496:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776858.466499:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466505:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776858.466506:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.466508:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776858.466510:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776858.466511:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776858.466514:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803776:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776858.466517:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776858.466519:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776858.466520:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.466521:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776858.466524:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776858.466526:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776858.466530:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776858.466532:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776858.466535:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776858.466538:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099803776, portal 4 00000100:00000001:2.0:1775776858.466540:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776858.466541:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741759744 : -114841967791872 : ffff978d454df100) 00000100:00000040:2.0:1775776858.466546:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466552:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776858.466555:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776858.466557:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776858.466571:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099803776, offset 0 00000400:00000200:2.0:1775776858.466575:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776858.466578:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.466581:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776858.466584:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776858.466586:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776858.466588:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776858.466591:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776858.466594:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4510:4510:256:4294967295] 192.168.201.139@tcp LPNI seq info [4510:4510:8:4294967295] 00000400:00000200:2.0:1775776858.466597:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.466603:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776858.466606:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.466609:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776858.466613:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776858.466615:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776858.466619:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776858.466621:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776858.466625:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.466626:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776858.466629:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776858.466632:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776858.466635:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776858.466637:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776858.466640:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776858.466643:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466656:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.466659:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466664:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.466666:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776858.466668:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776858.466671:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776858.466674:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776858.466676:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313246228 00000100:00000001:2.0:1775776858.466678:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313247482 00000100:00000001:2.0:1775776858.466680:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313246228 00000100:00000001:2.0:1775776858.466682:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313247482 00000100:00000001:2.0:1775776858.466683:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776858.466685:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.466687:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776858.466690:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776858/real 1775776858] req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776858.466697:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776858.466698:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776858.466700:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776858.466702:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776858.466703:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776858.466705:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776858.466708:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776858.466716:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466722:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776858.466724:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466726:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.466727:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776858.466730:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466737:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776858.466740:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466746:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776858.466748:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466749:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.466750:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.466752:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776858.466753:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776858.466755:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776858.466758:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776858.466760:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776858.466763:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776858.466771:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803776:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776858.466774:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776858.466777:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466802:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776858.466804:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.466806:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776858.466808:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776858.466809:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776858.466813:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803904:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776858.466815:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776858.466816:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776858.466818:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776858.466820:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776858.466823:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506000. 02000000:00000001:2.0:1775776858.466824:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776858.466827:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776858.466829:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776858.466832:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776858.466835:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099803904, portal 4 00000100:00000001:2.0:1775776858.466836:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776858.466838:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741761536 : -114841967790080 : ffff978d454df800) 00000100:00000040:2.0:1775776858.466842:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.466848:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776858.466850:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776858.466852:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776858.466854:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099803904, offset 0 00000400:00000200:2.0:1775776858.466857:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776858.466861:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.466864:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776858.466866:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776858.466869:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776858.466871:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776858.466873:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776858.466875:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4511:4511:256:4294967295] 192.168.201.139@tcp LPNI seq info [4511:4511:8:4294967295] 00000400:00000200:2.0:1775776858.466879:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776858.466884:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776858.466888:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.466891:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776858.467073:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776858.467076:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776858.467079:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776858.467081:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776858.467085:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776858.467087:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776858.467090:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776858.467093:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776858.467096:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776858.467098:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776858.467100:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776858.467104:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.467111:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.467114:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776883 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.467120:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.467122:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776858.467125:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776858.467128:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776858.467131:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776858.467133:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313246220 00000100:00000001:2.0:1775776858.467135:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313247482 00000100:00000001:2.0:1775776858.467137:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313246220 00000100:00000001:2.0:1775776858.467139:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313247482 00000100:00000001:2.0:1775776858.467141:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776858.467142:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.467144:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776858.467147:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776858/real 1775776858] req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776858.467154:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776858.467156:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776858.467157:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776858.467159:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776858.467161:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776858.467163:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776858.467165:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776858.467172:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.467176:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776858.467177:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.467179:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.467180:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776858.467183:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.467189:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776858.467192:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.467198:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776858.467199:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.467201:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776858.467202:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.467204:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776858.467206:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776858.467208:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776858.467211:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776858.467212:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776858.467215:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776858.471346:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099803904:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776858.471354:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.471358:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.471363:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd880 x1862033099803392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.471371:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776858.471374:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776858.471378:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776858.471383:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776858.471385:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776858.471387:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776858.471389:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776858.471391:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759cc00. 02000000:00000001:2.0:1775776858.471394:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776858.471396:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776858.471398:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd880. 00000100:00000001:2.0:1775776858.471402:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776858.471403:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.471405:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.471408:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df480 x1862033099803520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.471414:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776858.471415:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776858.471416:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776858.471418:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776858.471420:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776858.471421:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776858.471423:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776858.471424:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759ec00. 02000000:00000001:2.0:1775776858.471426:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776858.471427:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776858.471428:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df480. 00000100:00000001:2.0:1775776858.471430:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776858.471431:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.471432:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.471436:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ded80 x1862033099803648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.471442:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776858.471443:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776858.471445:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776858.471446:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776858.471447:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776858.471448:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776858.471449:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776858.471450:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:2.0:1775776858.471452:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776858.471452:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776858.471453:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776858.471455:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776858.471456:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.471458:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.471461:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df100 x1862033099803776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.471467:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776858.471468:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776858.471470:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776858.471472:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776858.471473:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776858.471474:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776858.471475:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776858.471476:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f400. 02000000:00000001:2.0:1775776858.471478:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776858.471479:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776858.471480:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df100. 00000100:00000001:2.0:1775776858.471481:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776858.471482:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.471483:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776858.471486:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df800 x1862033099803904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776883 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776858.471492:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776858.471493:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776858.471495:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506000. 02000000:00000001:2.0:1775776858.471496:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776858.471497:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776858.471498:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776858.471499:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776858.471500:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c000. 02000000:00000001:2.0:1775776858.471501:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776858.471502:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776858.471503:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df800. 00000100:00000001:2.0:1775776858.471505:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776858.471506:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.471508:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776858.471514:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776858.471516:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776858.471518:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776858.471519:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776859.164148:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776859.164154:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776859.164156:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776859.164162:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776859.164166:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776859.164170:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776859.164172:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776859.164175:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776859.164177:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776859.227996:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776859.228004:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776859.228007:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776859.228010:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776859.228014:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776859.228016:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776859.228018:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776859.228020:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776859.228023:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776859.228025:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776859.228028:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776859.228030:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776859.228033:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776859.228034:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776859.228233:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776859.228237:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776859.228241:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776859.355888:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776859.355895:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776859.355897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776859.355901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776859.355905:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776859.355909:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776859.355911:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776859.355913:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776859.355915:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776859.739937:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313247482, at 4313247488 00000100:00000010:1.0:1775776859.739953:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000040:0.0:1775776859.741185:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313247482, at 4313247489 00000100:00000010:0.0:1775776859.741195:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775776860.187904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776860.187911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776860.187913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776860.187919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:2.0:1775776860.187923:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776860.187927:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776860.187929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776860.187931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776860.187933:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776860.252047:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776860.252053:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776860.252059:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776860.252060:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776860.252063:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776860.252064:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776860.252068:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776860.252069:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776860.253622:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776860.253640:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776860.253645:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776860.379961:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776860.379971:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776860.379974:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776860.379981:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776860.379986:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776860.379991:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776860.379995:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776860.379997:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776860.380000:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776861.212709:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776861.212717:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776861.212718:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776861.212724:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776861.212727:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776861.212730:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776861.212733:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776861.212735:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776861.212738:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275962:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776861.275967:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275971:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776861.275974:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776861.275976:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776861.275978:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275980:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776861.275981:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275983:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275984:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275986:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776861.275987:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776861.275988:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776861.275989:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275989:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776861.275990:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275992:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275993:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275995:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776861.275996:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776861.275997:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776861.275997:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.275998:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776861.275999:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.276000:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776861.276001:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776861.276072:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776861.276075:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776861.276077:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776861.403912:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776861.403920:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776861.403921:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776861.403927:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776861.403930:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776861.403933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776861.403936:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776861.403938:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776861.403940:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776862.235890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776862.235896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776862.235898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776862.235903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776862.235907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776862.235911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776862.235913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776862.235915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776862.235918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776862.299955:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776862.299961:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776862.299968:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776862.299970:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776862.299973:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776862.299974:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776862.299977:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776862.299978:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776862.300232:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776862.300235:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776862.300238:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776862.427886:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776862.427891:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776862.427893:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776862.427898:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776862.427900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776862.427903:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776862.427905:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776862.427907:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776862.427909:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776863.259894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776863.259899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776863.259901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776863.259906:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776863.259915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776863.259918:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776863.259921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776863.259924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776863.259927:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776863.324105:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776863.324110:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776863.324116:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776863.324117:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776863.324121:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776863.324122:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776863.324125:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776863.324126:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776863.324188:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776863.324192:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776863.324195:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776863.451902:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776863.451908:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776863.451910:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776863.451914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776863.451918:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776863.451922:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776863.451924:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776863.451926:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776863.451929:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776863.579902:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776863.579914:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776863.579917:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776863.579921:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776863.579925:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776863.579933:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18578 00000400:00000200:0.0:1775776863.579942:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776863.579948:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776863.579952:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776863.579955:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.579958:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776863.579960:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.579963:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776863.579966:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776863.579968:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776863.579971:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776863.579973:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776863.579976:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776863.579978:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776863.579980:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.579982:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776863.579984:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.579986:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776863.579989:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776863.579992:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776863.579994:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776863.579997:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776863.580000:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776863.580003:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580005:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776863.580007:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776863.580016:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776863.580019:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776863.580022:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776863.580024:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580027:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776863.580030:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776863.580034:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776863.580042:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776863.580049:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776863.580058:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776863.580068:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776863.580072:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776863.580076:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776863.580098:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776863.580101:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776863.580104:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776863.580105:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776863.580107:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776863.580109:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776863.580111:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18578 00000400:00000200:0.0:1775776863.580114:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776863.580117:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776863.580119:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776863.580122:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580124:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776863.580127:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580129:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776863.580131:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776863.580133:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776863.580134:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776863.580136:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776863.580138:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776863.580140:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776863.580140:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776863.580145:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776863.580145:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776863.580149:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776863.580149:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776863.580153:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580156:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000040:2.0:1775776863.580159:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776863.580160:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776863.580166:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580172:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776863.580176:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776863.580177:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776863.580180:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776863.580182:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776863.580183:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776863.580186:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776863.580187:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580191:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775776863.580191:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776863.580197:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775776863.580198:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099804032:192.168.201.139@tcp:250:kworker.0 00000008:00000001:0.0:1775776863.580202:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580205:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776863.580206:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.580210:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:0.0:1775776863.580211:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 02000000:00000001:2.0:1775776863.580213:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776863.580214:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776863.580217:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775776863.580219:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776863.580221:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776863.580222:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 00000100:00000001:0.0:1775776863.580223:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776863.580226:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776863.580227:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776863.580229:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776863.580234:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000010:0.0:1775776863.580235:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775776863.580240:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776863.580240:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000100:00100000:0.0:1775776863.580248:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775776863.580248:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000001:0.0:1775776863.580258:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1775776863.580258:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099804032, portal 25 00000100:00000001:0.0:1775776863.580262:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580263:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:0.0:1775776863.580267:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.580267:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757952 : -114841967793664 : ffff978d454dea00) 00000100:00000040:2.0:1775776863.580277:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776863.580279:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776863.580284:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776863.580293:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776863.580295:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776863.580296:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776863.580299:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:2.0:1775776863.580300:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580303:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:2.0:1775776863.580305:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00080000:0.0:1775776863.580308:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18578 00000100:00000200:2.0:1775776863.580310:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099804032, offset 0 00000400:00000200:0.0:1775776863.580316:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776863.580322:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000200:2.0:1775776863.580322:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00080000:0.0:1775776863.580327:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776863.580331:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776863.580334:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776863.580335:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776863.580339:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580342:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776863.580345:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000020:00000001:0.0:1775776863.580346:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776863.580348:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776863.580351:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776863.580353:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000400:00000200:2.0:1775776863.580354:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1775776863.580356:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776863.580359:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776863.580360:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000040:0.0:1775776863.580366:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776863.580367:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775776863.580371:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776863.580373:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000040:0.0:1775776863.580376:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580380:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776863.580380:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4512:4512:256:4294967295] 192.168.201.139@tcp LPNI seq info [4512:4512:8:4294967295] 00000020:00000040:0.0:1775776863.580383:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:2.0:1775776863.580389:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:0.0:1775776863.580390:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580394:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776863.580396:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776863.580400:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776863.580405:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776863.580405:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000001:0.0:1775776863.580409:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776863.580413:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.0:1775776863.580416:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000008:00000001:0.0:1775776863.580418:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580420:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776863.580422:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000100:00000010:0.0:1775776863.580425:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776863.580429:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000200:2.0:1775776863.580431:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776863.580432:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776863.580435:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580437:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.0:1775776863.580437:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 02000000:00000001:0.0:1775776863.580439:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776863.580441:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776863.580446:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000800:00000200:2.1:1775776863.580446:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776863.580452:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776863.580452:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775776863.580461:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00100000:0.0:1775776863.580462:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:2.0:1775776863.580464:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000100:00000001:0.0:1775776863.580483:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776863.580485:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776863.580490:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776863.580490:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776863.580500:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775776863.580506:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776863.580507:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000100:00080000:0.0:1775776863.580510:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1775776863.580511:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00080000:0.0:1775776863.580516:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776863.580517:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775776863.580519:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776863.580521:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1775776863.580524:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776863.580525:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776863.580531:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18578 00000400:00000200:0.0:1775776863.580538:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776863.580538:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.580544:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776863.580545:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776863.580549:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776863.580555:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580560:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776863.580561:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776863.580563:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776863.580565:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776863.580568:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776863.580570:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776863.580572:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000100:00000001:2.0:1775776863.580572:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776863.580575:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776863.580577:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776863.580580:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000400:00000200:2.0:1775776863.580581:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:0.0:1775776863.580585:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776863.580587:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313252597 00000100:00000040:0.0:1775776863.580590:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580593:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776863.580595:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000010:2.0:1775776863.580595:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b1e0. 00000100:00000001:0.0:1775776863.580600:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776863.580603:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000001:2.0:1775776863.580604:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1775776863.580608:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580612:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580613:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313252597 02000000:00000001:0.0:1775776863.580614:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:2.0:1775776863.580619:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc60. 02000000:00000001:0.0:1775776863.580620:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776863.580624:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580625:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776863.580626:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776863.580630:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1775776863.580632:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775776863.580634:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580636:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580638:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580639:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000010:0.0:1775776863.580642:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776863.580647:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00100000:2.0:1775776863.580647:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776863/real 1775776863] req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776863.580652:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776863.580654:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580658:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776863.580660:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776863.580663:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1775776863.580666:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:0.0:1775776863.580668:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000400:00000001:2.0:1775776863.580668:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776863.580672:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000100:00000001:0.0:1775776863.580673:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776863.580676:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776863.580680:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00100000:0.0:1775776863.580681:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776863.580683:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776863.580690:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776863.580694:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776863.580698:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776863.580707:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776863.580708:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776863.580736:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580736:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776863.580738:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776863.580740:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776863.580742:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1775776863.580745:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776863.580745:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776863.580748:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776863.580752:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775776863.580752:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776863.580758:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776863.580762:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18578 00000400:00000200:0.0:1775776863.580769:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776863.580769:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776863.580775:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776863.580778:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776863.580800:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580802:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776863.580804:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580806:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776863.580808:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776863.580810:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776863.580811:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776863.580813:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776863.580815:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776863.580818:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776863.580819:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776863.580821:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580826:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776863.580830:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776863.580835:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776863.580837:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:0.0:1775776863.580839:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00000001:2.0:1775776863.580841:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776863.580844:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776863.580845:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.580847:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776863.580849:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580851:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1775776863.580853:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776863.580854:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1775776863.580856:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1775776863.580858:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1775776863.580859:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580863:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776863.580865:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1775776863.580866:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776863.580870:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:0.0:1775776863.580871:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776863.580875:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776863.580878:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776863.580880:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776863.580885:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776863.580890:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776863.580893:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776863.580895:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099804032:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776863.580896:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776863.580899:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776863.580903:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580905:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1775776863.580907:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:2.0:1775776863.580908:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.580912:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dea00 x1862033099804032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776863.580913:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776863.580922:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776863.580923:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776863.580926:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776863.580930:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 00000100:00000001:0.0:1775776863.580934:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1775776863.580934:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776863.580937:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1775776863.580938:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776863.580939:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776863.580943:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000040:0.0:1775776863.580944:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775776863.580946:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:2.0:1775776863.580953:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1775776863.580955:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.580956:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776863.580959:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dea00. 00000100:00000040:0.0:1775776863.580961:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18589) 00000100:00000001:2.0:1775776863.580962:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776863.580965:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.580968:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.580976:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776863.580978:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.580980:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776863.580982:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776863.580984:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776863.580988:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.580995:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776863.580996:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.580998:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776863.581000:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776863.581002:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776863.581005:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804160:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776863.581008:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776863.581010:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776863.581011:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.581013:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776863.581016:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776863.581018:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776863.581021:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776863.581024:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776863.581027:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776863.581030:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099804160, portal 10 00000100:00000001:2.0:1775776863.581032:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776863.581033:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741752576 : -114841967799040 : ffff978d454dd500) 00000100:00000040:2.0:1775776863.581039:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581045:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776863.581048:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776863.581049:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776863.581052:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099804160, offset 0 00000400:00000200:2.0:1775776863.581055:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776863.581059:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776863.581062:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776863.581064:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776863.581066:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776863.581069:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776863.581071:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776863.581074:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4513:4513:256:4294967295] 192.168.201.139@tcp LPNI seq info [4513:4513:8:4294967295] 00000400:00000200:2.0:1775776863.581078:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776863.581084:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776863.581089:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776863.581092:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776863.581095:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776863.581098:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776863.581102:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776863.581105:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775776863.581109:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776863.581111:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776863.581114:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776863.581118:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776863.581120:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776863.581122:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776863.581124:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776863.581127:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581134:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.581137:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581144:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.581146:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776863.581149:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776863.581153:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776863.581156:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776863.581158:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313251333 00000100:00000001:2.0:1775776863.581160:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313252597 00000100:00000001:2.0:1775776863.581162:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313251333 00000100:00000001:2.0:1775776863.581164:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313252597 00000100:00000001:2.0:1775776863.581166:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776863.581168:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.581169:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776863.581172:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776863/real 1775776863] req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776863.581179:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776863.581181:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776863.581183:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776863.581185:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776863.581186:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776863.581188:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776863.581192:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776863.581199:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581205:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776863.581206:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581208:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.581209:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776863.581212:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581218:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776863.581221:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581227:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776863.581230:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581231:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.581232:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581234:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776863.581236:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776863.581238:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776863.581241:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776863.581242:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776863.581246:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776863.581253:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804160:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776863.581256:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776863.581259:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581271:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776863.581272:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.581274:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776863.581276:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776863.581278:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776863.581280:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804288:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776863.581284:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776863.581285:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776863.581287:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.581288:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776863.581291:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776863.581293:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776863.581296:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776863.581298:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776863.581301:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776863.581303:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099804288, portal 10 00000100:00000001:2.0:1775776863.581305:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776863.581306:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741761536 : -114841967790080 : ffff978d454df800) 00000100:00000040:2.0:1775776863.581311:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581317:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776863.581319:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776863.581321:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776863.581324:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099804288, offset 0 00000400:00000200:2.0:1775776863.581328:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776863.581332:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776863.581335:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776863.581337:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776863.581340:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776863.581342:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776863.581344:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776863.581348:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4514:4514:256:4294967295] 192.168.201.139@tcp LPNI seq info [4514:4514:8:4294967295] 00000400:00000200:2.0:1775776863.581352:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776863.581358:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776863.581362:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776863.581365:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776863.581369:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776863.581371:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776863.581375:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776863.581377:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775776863.581381:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776863.581383:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776863.581385:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776863.581388:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776863.581391:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776863.581392:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776863.581394:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776863.581398:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581405:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.581408:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581414:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.581415:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776863.581418:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776863.581420:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776863.581423:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776863.581425:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313251334 00000100:00000001:2.0:1775776863.581427:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313252597 00000100:00000001:2.0:1775776863.581429:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313251334 00000100:00000001:2.0:1775776863.581431:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313252597 00000100:00000001:2.0:1775776863.581433:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776863.581435:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.581436:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776863.581440:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776863/real 1775776863] req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776863.581447:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776863.581448:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776863.581450:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776863.581452:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776863.581454:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776863.581456:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776863.581458:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776863.581467:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581484:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776863.581485:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581487:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.581488:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776863.581491:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581498:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776863.581502:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581508:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776863.581511:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581512:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.581513:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581515:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776863.581516:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776863.581518:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776863.581522:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776863.581524:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776863.581528:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776863.581536:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804288:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776863.581540:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776863.581543:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581549:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776863.581550:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.581552:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776863.581554:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776863.581556:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776863.581558:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804416:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776863.581561:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776863.581563:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776863.581565:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.581566:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776863.581570:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776863.581572:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776863.581576:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776863.581578:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776863.581581:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776863.581584:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099804416, portal 4 00000100:00000001:2.0:1775776863.581585:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776863.581587:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741759744 : -114841967791872 : ffff978d454df100) 00000100:00000040:2.0:1775776863.581592:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581598:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776863.581601:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776863.581603:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776863.581605:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099804416, offset 0 00000400:00000200:2.0:1775776863.581608:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776863.581612:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776863.581615:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776863.581618:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776863.581620:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776863.581622:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776863.581625:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776863.581627:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4515:4515:256:4294967295] 192.168.201.139@tcp LPNI seq info [4515:4515:8:4294967295] 00000400:00000200:2.0:1775776863.581632:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776863.581638:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776863.581641:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776863.581644:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776863.581647:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776863.581650:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776863.581654:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776863.581656:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775776863.581659:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776863.581661:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776863.581664:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776863.581667:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776863.581669:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776863.581671:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776863.581673:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776863.581676:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581683:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.581686:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581692:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.581694:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776863.581697:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776863.581699:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776863.581702:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776863.581704:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313251343 00000100:00000001:2.0:1775776863.581706:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313252597 00000100:00000001:2.0:1775776863.581708:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313251343 00000100:00000001:2.0:1775776863.581710:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313252597 00000100:00000001:2.0:1775776863.581712:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776863.581713:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.581715:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776863.581718:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776863/real 1775776863] req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776863.581725:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776863.581727:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776863.581728:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776863.581730:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776863.581732:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776863.581734:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776863.581737:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776863.581745:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581751:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776863.581753:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581755:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.581756:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776863.581759:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581765:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776863.581768:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581775:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776863.581795:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581797:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.581798:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.581800:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776863.581802:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776863.581804:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776863.581807:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776863.581809:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776863.581813:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776863.581821:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804416:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776863.581825:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776863.581828:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581833:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776863.581835:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.581837:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776863.581838:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776863.581840:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776863.581842:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804544:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776863.581846:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776863.581847:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776863.581849:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776863.581850:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776863.581853:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776863.581855:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776863.581858:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776863.581860:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776863.581863:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776863.581865:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099804544, portal 4 00000100:00000001:2.0:1775776863.581867:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776863.581869:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741758848 : -114841967792768 : ffff978d454ded80) 00000100:00000040:2.0:1775776863.581874:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581880:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776863.581882:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776863.581884:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776863.581886:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099804544, offset 0 00000400:00000200:2.0:1775776863.581889:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776863.581893:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776863.581895:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776863.581898:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776863.581900:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776863.581902:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776863.581905:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776863.581908:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4516:4516:256:4294967295] 192.168.201.139@tcp LPNI seq info [4516:4516:8:4294967295] 00000400:00000200:2.0:1775776863.581911:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776863.581917:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776863.581922:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776863.581925:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776863.581928:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776863.581931:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1775776863.581935:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776863.581938:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1775776863.581941:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776863.581943:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776863.581946:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776863.581948:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776863.581951:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776863.581952:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776863.581955:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776863.581958:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581965:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.581968:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776888 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.581974:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.581975:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776863.581977:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776863.581980:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776863.581982:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776863.581984:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313251335 00000100:00000001:2.0:1775776863.581986:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313252597 00000100:00000001:2.0:1775776863.581988:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313251335 00000100:00000001:2.0:1775776863.581990:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313252597 00000100:00000001:2.0:1775776863.581992:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776863.581994:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.581996:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776863.581999:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776863/real 1775776863] req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776863.582006:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776863.582007:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776863.582009:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776863.582011:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776863.582012:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776863.582014:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776863.582017:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776863.582025:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.582036:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776863.582038:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582039:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.582040:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776863.582043:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.582050:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776863.582053:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.582059:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776863.582061:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582062:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776863.582064:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582066:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776863.582067:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776863.582070:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776863.582072:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776863.582075:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776863.582078:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776863.582086:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804544:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776863.582089:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582090:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.582093:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd500 x1862033099804160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.582099:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776863.582101:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776863.582103:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776863.582105:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776863.582107:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776863.582108:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776863.582110:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776863.582112:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:2.0:1775776863.582115:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776863.582116:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776863.582118:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd500. 00000100:00000001:2.0:1775776863.582120:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776863.582121:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582123:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.582126:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df800 x1862033099804288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.582132:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776863.582134:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776863.582135:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776863.582137:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776863.582138:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776863.582139:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776863.582140:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776863.582142:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:2.0:1775776863.582144:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776863.582145:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776863.582146:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df800. 00000100:00000001:2.0:1775776863.582148:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776863.582149:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582151:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.582153:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df100 x1862033099804416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.582159:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776863.582160:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776863.582161:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776863.582163:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776863.582163:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776863.582164:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776863.582167:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776863.582168:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d000. 02000000:00000001:2.0:1775776863.582170:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776863.582171:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776863.582172:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df100. 00000100:00000001:2.0:1775776863.582174:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776863.582175:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582176:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776863.582179:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ded80 x1862033099804544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776888 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776863.582184:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776863.582185:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776863.582187:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776863.582188:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776863.582189:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776863.582190:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776863.582192:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776863.582193:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759dc00. 02000000:00000001:2.0:1775776863.582194:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776863.582195:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776863.582196:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776863.582198:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776863.582199:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582201:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776863.582205:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776863.582206:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776863.582208:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776863.582210:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776864.283908:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776864.283916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776864.283918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776864.283923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776864.283927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776864.283931:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776864.283934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776864.283937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776864.283939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776864.347940:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776864.347946:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776864.347953:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776864.347954:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776864.347957:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776864.347958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776864.347961:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776864.347961:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776864.348033:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776864.348036:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776864.348038:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776864.477226:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776864.477233:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776864.477235:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776864.477241:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776864.477245:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776864.477248:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776864.477251:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776864.477253:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776864.477256:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776864.860013:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313252597, at 4313252608 00000100:00000010:1.0:1775776864.860027:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000040:0.0:1775776864.860312:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313252597, at 4313252608 00000100:00000010:0.0:1775776864.860318:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:3.0:1775776865.307912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776865.307919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776865.307921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776865.307926:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776865.307930:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776865.307933:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776865.307935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776865.307938:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776865.307940:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776865.371893:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776865.371899:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776865.371903:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776865.371904:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776865.371906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776865.371907:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776865.371909:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776865.371911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776865.371969:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776865.371972:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776865.371975:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776865.500713:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776865.500720:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776865.500722:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776865.500727:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776865.500732:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776865.500736:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776865.500738:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776865.500740:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776865.500744:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776866.331901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776866.331908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776866.331910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776866.331915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776866.331919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776866.331923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776866.331925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776866.331927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776866.331930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776866.398598:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776866.398605:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776866.398610:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776866.398611:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776866.398614:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776866.398615:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776866.398619:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776866.398621:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776866.398688:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776866.398692:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776866.398695:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776866.529174:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776866.529181:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776866.529183:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776866.529187:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776866.529191:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776866.529194:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776866.529197:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776866.529200:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776866.529203:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776867.355921:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776867.355928:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776867.355930:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776867.355934:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776867.355938:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776867.355942:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776867.355944:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776867.355947:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776867.355950:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776867.419893:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776867.419898:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776867.419902:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776867.419903:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776867.419905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776867.419906:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776867.419908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776867.419909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776867.420007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776867.420009:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776867.420011:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776867.547898:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776867.547905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776867.547907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776867.547913:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776867.547916:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776867.547920:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776867.547924:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776867.547927:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776867.547929:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.379906:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776868.379913:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776868.379914:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776868.379920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776868.379924:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776868.379928:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776868.379931:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776868.379933:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776868.379936:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776868.446879:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776868.446886:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776868.446891:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776868.446892:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776868.446895:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776868.446896:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776868.446899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776868.446900:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776868.447014:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776868.447017:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776868.447019:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776868.571892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776868.571900:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776868.571902:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776868.571908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776868.571911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776868.571916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776868.571918:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776868.571921:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776868.571923:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776868.700552:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776868.700561:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776868.700564:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776868.700569:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776868.700573:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776868.700580:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18584 00000400:00000200:3.0:1775776868.700590:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776868.700596:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776868.700599:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776868.700602:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700605:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.700607:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700609:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776868.700613:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776868.700615:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776868.700618:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776868.700620:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776868.700623:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776868.700625:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776868.700627:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700629:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.700630:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700632:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776868.700634:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776868.700636:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700638:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776868.700641:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776868.700644:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776868.700646:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.700648:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776868.700650:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776868.700659:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776868.700662:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776868.700665:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776868.700667:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.700669:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776868.700672:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776868.700676:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776868.700683:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776868.700690:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776868.700698:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776868.700708:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776868.700712:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776868.700717:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776868.700745:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776868.700747:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776868.700751:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776868.700752:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776868.700753:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776868.700755:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776868.700757:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18584 00000400:00000200:3.0:1775776868.700760:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776868.700763:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776868.700765:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776868.700767:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700769:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.700771:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700772:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776868.700774:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776868.700775:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776868.700777:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776868.700808:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776868.700811:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776868.700813:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776868.700816:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700818:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.700819:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700821:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776868.700823:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776868.700826:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700828:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776868.700830:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776868.700835:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776868.700837:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.700839:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776868.700843:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776868.700846:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.700847:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776868.700850:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776868.700853:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776868.700855:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776868.700856:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.700857:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776868.700858:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776868.700860:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776868.700863:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776868.700866:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776868.700870:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776868.700877:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776868.700878:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776868.700881:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776868.700887:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776868.700889:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776868.700892:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776868.700893:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776868.700894:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776868.700896:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776868.700898:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18584 00000400:00000200:3.0:1775776868.700901:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776868.700904:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776868.700905:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776868.700908:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700909:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.700911:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700912:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776868.700914:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776868.700915:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776868.700916:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776868.700918:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776868.700920:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776868.700921:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776868.700923:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700925:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.700926:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700928:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776868.700930:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776868.700933:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776868.700935:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776868.700936:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776868.700939:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776868.700940:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.700943:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776868.700944:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776868.700946:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.700948:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776868.700951:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776868.700953:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776868.700955:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776868.700957:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.700958:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776868.700959:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776868.700961:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776868.700964:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776868.700967:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776868.700971:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776868.700978:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776868.700980:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776868.700984:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776868.700990:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776868.700993:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776868.700996:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776868.700998:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776868.700999:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776868.701002:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776868.701004:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18584 00000400:00000200:3.0:1775776868.701008:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776868.701010:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776868.701012:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776868.701015:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701017:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.701019:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701020:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776868.701023:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776868.701024:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776868.701025:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776868.701027:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776868.701029:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776868.701031:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776868.701034:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701035:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.701037:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701039:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776868.701041:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776868.701044:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701046:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776868.701048:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776868.701050:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776868.701052:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.701054:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776868.701056:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776868.701058:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.701060:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776868.701063:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776868.701065:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776868.701067:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776868.701068:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.701070:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776868.701071:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776868.701073:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776868.701076:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776868.701079:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776868.701083:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776868.701089:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776868.701091:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776868.701094:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776868.701102:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776868.701104:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776868.701108:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776868.701109:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776868.701111:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776868.701113:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776868.701115:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18584 00000400:00000200:3.0:1775776868.701118:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776868.701121:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776868.701123:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776868.701126:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701127:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.701129:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701131:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776868.701133:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776868.701134:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776868.701135:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776868.701137:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776868.701139:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776868.701141:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776868.701143:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701145:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776868.701146:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701148:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776868.701150:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776868.701153:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776868.701155:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776868.701157:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776868.701159:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776868.701161:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.701163:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776868.701165:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776868.701167:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.701169:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776868.701173:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776868.701175:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776868.701177:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776868.701178:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776868.701180:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776868.701181:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776868.701182:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776868.701185:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776868.701188:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776868.701192:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776868.701198:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776868.701200:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776868.701203:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776868.701210:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776868.701214:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18594) 00000100:00000001:2.0:1775776868.701310:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776868.701314:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776868.701316:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776868.701321:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701328:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776868.701331:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.701334:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776868.701336:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776868.701338:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776868.701342:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099804672:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776868.701346:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776868.701350:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776868.701352:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.701354:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776868.701359:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776868.701361:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776868.701366:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776868.701369:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776868.701376:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776868.701395:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099804672, portal 25 00000100:00000001:2.0:1775776868.701398:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776868.701399:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751254400 : -114841958297216 : ffff978d45ded180) 00000100:00000040:2.0:1775776868.701404:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701411:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776868.701414:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776868.701417:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776868.701420:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099804672, offset 0 00000400:00000200:2.0:1775776868.701426:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776868.701433:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.701437:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776868.701442:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776868.701445:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776868.701448:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776868.701451:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776868.701455:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4517:4517:256:4294967295] 192.168.201.139@tcp LPNI seq info [4517:4517:8:4294967295] 00000400:00000200:2.0:1775776868.701458:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.701466:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776868.701472:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.701477:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776868.701483:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776868.701486:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776868.701491:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776868.701494:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776868.701498:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.701500:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776868.701505:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776868.701509:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776868.701514:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776868.701517:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776868.701521:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776868.701525:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701533:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.701536:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701543:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.701544:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776868.701549:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776868.701552:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776868.701555:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313257718 00000100:00000010:2.0:1775776868.701559:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc00. 00000100:00000001:2.0:1775776868.701566:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776868.701613:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313257718 00000100:00000010:2.0:1775776868.701617:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bcc0. 00000100:00000001:2.0:1775776868.701621:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776868.701624:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776868.701627:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.701629:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776868.701633:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776868/real 1775776868] req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776868.701647:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776868.701648:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776868.701650:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776868.701652:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776868.701654:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776868.701656:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776868.701659:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776868.701666:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701673:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776868.701674:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.701676:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.701678:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776868.701681:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701687:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776868.701691:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701697:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776868.701699:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.701701:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.701702:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.701703:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776868.701705:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776868.701708:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776868.701711:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776868.701713:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776868.701717:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776868.701723:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099804672:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776868.701727:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776868.701729:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701735:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776868.701736:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.701738:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776868.701739:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776868.701741:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776868.701743:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776868.701745:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776868.701747:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776868.701748:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.701750:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776868.701753:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776868.701755:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776868.701759:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776868.701761:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776868.701764:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776868.701766:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099804800, portal 10 00000100:00000001:2.0:1775776868.701768:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776868.701769:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751260672 : -114841958290944 : ffff978d45deea00) 00000100:00000040:2.0:1775776868.701774:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701805:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776868.701807:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776868.701809:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776868.701812:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099804800, offset 0 00000400:00000200:2.0:1775776868.701816:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776868.701820:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.701823:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776868.701825:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776868.701828:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776868.701830:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776868.701832:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776868.701834:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4518:4518:256:4294967295] 192.168.201.139@tcp LPNI seq info [4518:4518:8:4294967295] 00000400:00000200:2.0:1775776868.701838:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.701843:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776868.701847:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.701850:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776868.701853:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776868.701855:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776868.701859:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776868.701860:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776868.701864:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.701865:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776868.701867:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776868.701870:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776868.701872:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776868.701873:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776868.701875:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776868.701878:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701884:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.701886:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701891:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.701893:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776868.701895:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776868.701898:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776868.701901:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776868.701903:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313256454 00000100:00000001:2.0:1775776868.701905:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313257718 00000100:00000001:2.0:1775776868.701908:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313256454 00000100:00000001:2.0:1775776868.701909:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313257718 00000100:00000001:2.0:1775776868.701911:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776868.701912:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.701914:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776868.701917:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776868/real 1775776868] req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776868.701923:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776868.701924:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776868.701925:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776868.701928:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776868.701929:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776868.701931:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776868.701934:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776868.701940:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701945:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776868.701947:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.701948:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.701949:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776868.701952:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701958:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776868.701961:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.701967:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776868.701970:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.701971:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.701972:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.701974:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776868.701975:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776868.701978:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776868.701981:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776868.701983:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776868.701986:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776868.701994:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776868.701997:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776868.702000:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702006:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776868.702007:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.702009:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776868.702010:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776868.702012:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776868.702015:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776868.702018:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776868.702019:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776868.702021:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.702022:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776868.702025:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776868.702027:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776868.702031:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776868.702033:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776868.702036:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776868.702038:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099804928, portal 10 00000100:00000001:2.0:1775776868.702041:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776868.702042:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751250816 : -114841958300800 : ffff978d45dec380) 00000100:00000040:2.0:1775776868.702046:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702052:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776868.702055:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776868.702057:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776868.702059:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099804928, offset 0 00000400:00000200:2.0:1775776868.702062:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776868.702066:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.702069:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776868.702071:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776868.702073:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776868.702076:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776868.702078:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776868.702080:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4519:4519:256:4294967295] 192.168.201.139@tcp LPNI seq info [4519:4519:8:4294967295] 00000400:00000200:2.0:1775776868.702084:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.702089:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776868.702093:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.702096:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776868.702099:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776868.702101:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776868.702105:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776868.702107:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776868.702110:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.702112:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776868.702115:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776868.702118:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776868.702120:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776868.702122:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776868.702124:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776868.702127:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702139:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.702142:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702147:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.702149:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776868.702151:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776868.702153:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776868.702156:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776868.702157:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313256455 00000100:00000001:2.0:1775776868.702159:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313257718 00000100:00000001:2.0:1775776868.702161:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313256455 00000100:00000001:2.0:1775776868.702163:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313257718 00000100:00000001:2.0:1775776868.702164:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776868.702166:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.702167:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776868.702170:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776868/real 1775776868] req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776868.702178:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776868.702179:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776868.702180:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776868.702182:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776868.702184:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776868.702185:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776868.702188:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776868.702193:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702198:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776868.702200:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702201:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.702203:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776868.702206:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702212:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776868.702216:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702222:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776868.702224:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702225:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.702226:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702227:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776868.702229:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776868.702231:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776868.702234:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776868.702236:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776868.702240:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776868.702247:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099804928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776868.702251:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776868.702254:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702260:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776868.702262:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.702263:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776868.702265:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776868.702267:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776868.702270:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776868.702273:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776868.702275:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776868.702276:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.702278:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776868.702281:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776868.702283:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776868.702286:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776868.702288:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776868.702291:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776868.702293:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099805056, portal 4 00000100:00000001:2.0:1775776868.702295:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776868.702296:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751252608 : -114841958299008 : ffff978d45deca80) 00000100:00000040:2.0:1775776868.702301:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702306:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776868.702308:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776868.702310:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776868.702312:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099805056, offset 0 00000400:00000200:2.0:1775776868.702315:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776868.702318:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.702321:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776868.702323:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776868.702325:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776868.702327:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776868.702329:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776868.702332:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4520:4520:256:4294967295] 192.168.201.139@tcp LPNI seq info [4520:4520:8:4294967295] 00000400:00000200:2.0:1775776868.702335:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.702341:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776868.702345:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.702348:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776868.702351:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776868.702354:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776868.702357:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776868.702359:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776868.702363:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.702365:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776868.702367:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776868.702370:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776868.702372:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776868.702373:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776868.702375:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776868.702378:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702397:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.702400:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702406:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.702408:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776868.702411:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776868.702414:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776868.702416:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776868.702418:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313256464 00000100:00000001:2.0:1775776868.702420:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313257718 00000100:00000001:2.0:1775776868.702422:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313256464 00000100:00000001:2.0:1775776868.702424:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313257718 00000100:00000001:2.0:1775776868.702426:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776868.702427:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.702429:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776868.702432:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776868/real 1775776868] req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776868.702438:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776868.702440:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776868.702442:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776868.702444:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776868.702445:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776868.702447:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776868.702450:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776868.702458:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702463:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776868.702464:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702466:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.702467:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776868.702469:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702475:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776868.702478:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702483:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776868.702486:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702487:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.702488:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702490:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776868.702491:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776868.702493:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776868.702496:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776868.702498:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776868.702501:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776868.702513:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776868.702516:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776868.702519:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702525:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776868.702526:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.702528:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776868.702529:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776868.702531:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776868.702534:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776868.702536:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776868.702538:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776868.702539:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776868.702541:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776868.702544:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775776868.702546:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776868.702549:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776868.702551:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776868.702554:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776868.702556:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099805184, portal 4 00000100:00000001:2.0:1775776868.702557:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776868.702558:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751259776 : -114841958291840 : ffff978d45dee680) 00000100:00000040:2.0:1775776868.702562:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702568:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776868.702570:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776868.702571:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776868.702573:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099805184, offset 0 00000400:00000200:2.0:1775776868.702577:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776868.702580:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.702584:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776868.702586:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776868.702588:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776868.702591:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776868.702593:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776868.702596:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4521:4521:256:4294967295] 192.168.201.139@tcp LPNI seq info [4521:4521:8:4294967295] 00000400:00000200:2.0:1775776868.702600:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776868.702606:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776868.702610:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.702613:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776868.702617:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776868.702619:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1775776868.702623:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776868.702626:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1775776868.702630:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776868.702631:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776868.702634:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776868.702637:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776868.702640:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776868.702641:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776868.702643:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776868.702646:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702653:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.702656:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776893 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702661:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.702663:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776868.702665:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776868.702668:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776868.702671:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776868.702673:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313256455 00000100:00000001:2.0:1775776868.702675:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313257718 00000100:00000001:2.0:1775776868.702677:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313256455 00000100:00000001:2.0:1775776868.702679:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313257718 00000100:00000001:2.0:1775776868.702681:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776868.702683:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.702684:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776868.702688:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776868/real 1775776868] req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776868.702695:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776868.702696:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776868.702697:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776868.702699:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776868.702700:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776868.702702:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776868.702704:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776868.702711:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702717:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776868.702718:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702720:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.702721:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776868.702725:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702731:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776868.702734:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702741:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776868.702743:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702745:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776868.702746:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702747:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776868.702749:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776868.702751:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776868.702754:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776868.702756:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776868.702760:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776868.702766:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776868.702769:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702771:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.702774:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099804672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702801:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776868.702804:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776868.702806:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776868.702809:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776868.702811:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776868.702813:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776868.702815:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776868.702817:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:2.0:1775776868.702820:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776868.702822:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776868.702823:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:2.0:1775776868.702826:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776868.702827:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702828:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.702832:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099804800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702842:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776868.702843:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776868.702845:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776868.702847:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776868.702848:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776868.702849:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776868.702850:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776868.702851:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:2.0:1775776868.702853:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776868.702854:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776868.702855:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:2.0:1775776868.702858:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776868.702859:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702860:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.702863:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec380 x1862033099804928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702868:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776868.702869:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776868.702871:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776868.702873:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776868.702874:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776868.702875:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776868.702877:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776868.702878:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:2.0:1775776868.702879:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776868.702880:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776868.702881:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec380. 00000100:00000001:2.0:1775776868.702883:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776868.702884:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702885:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.702888:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deca80 x1862033099805056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702893:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776868.702893:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776868.702895:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776868.702897:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776868.702898:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776868.702899:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776868.702901:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776868.702902:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:2.0:1775776868.702904:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776868.702905:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776868.702906:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deca80. 00000100:00000001:2.0:1775776868.702908:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776868.702909:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702910:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776868.702912:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee680 x1862033099805184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776893 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776868.702918:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776868.702919:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776868.702920:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775776868.702922:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776868.702923:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776868.702924:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776868.702925:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776868.702926:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:2.0:1775776868.702928:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776868.702929:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776868.702930:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee680. 00000100:00000001:2.0:1775776868.702932:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776868.702933:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702935:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776868.702941:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776868.702943:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776868.702945:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776868.702946:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776869.403918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776869.403925:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776869.403927:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776869.403933:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776869.403937:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776869.403941:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776869.403943:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776869.403945:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776869.403948:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776869.468023:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776869.468030:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776869.468032:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776869.468036:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776869.468040:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776869.468043:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776869.468045:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776869.468047:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776869.468051:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776869.468052:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776869.468056:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776869.468057:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776869.468060:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776869.468061:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776869.468834:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776869.468839:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776869.468842:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776869.595903:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776869.595910:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776869.595912:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776869.595917:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776869.595921:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776869.595924:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776869.595927:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776869.595929:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776869.595932:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776869.979933:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313257718, at 4313257728 00000100:00000010:1.0:1775776869.979958:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000040:0.0:1775776869.980161:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313257718, at 4313257728 00000100:00000010:0.0:1775776869.980168:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000001:3.0:1775776870.428111:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776870.428119:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776870.428120:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776870.428126:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776870.428129:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776870.428133:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776870.428135:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776870.428137:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776870.428140:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776870.491919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776870.491925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776870.491929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776870.491931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776870.491934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776870.491936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776870.491939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776870.491940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776870.492042:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776870.492045:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776870.492048:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776870.619890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776870.619898:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776870.619900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776870.619906:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776870.619909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776870.619912:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776870.619915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776870.619917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776870.619919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776871.452110:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776871.452116:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776871.452118:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776871.452122:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776871.452125:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776871.452128:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776871.452129:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776871.452131:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776871.452133:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516422:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776871.516428:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516433:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776871.516436:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776871.516439:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776871.516442:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516446:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776871.516448:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516449:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516452:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516455:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776871.516456:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776871.516458:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776871.516459:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516461:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776871.516462:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516463:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516465:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516468:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776871.516470:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776871.516471:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776871.516472:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516474:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776871.516475:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516477:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776871.516479:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776871.516565:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776871.516568:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776871.516571:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776871.643897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776871.643903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776871.643904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776871.643908:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776871.643911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776871.643914:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776871.643916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776871.643918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776871.643920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776872.475961:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776872.475968:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776872.475971:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776872.475977:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776872.475980:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776872.475985:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776872.475987:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776872.475989:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776872.475992:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776872.539911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776872.539918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776872.539925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776872.539926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776872.539929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776872.539930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776872.539933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776872.539935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776872.540151:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776872.540154:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776872.540157:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776872.668425:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776872.668432:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776872.668434:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776872.668438:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776872.668442:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776872.668445:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776872.668448:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776872.668450:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776872.668452:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776873.499902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776873.499909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776873.499911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776873.499917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776873.499920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776873.499924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776873.499927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776873.499929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776873.499932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776873.565487:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776873.565494:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776873.565500:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776873.565501:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776873.565504:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776873.565505:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776873.565508:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776873.565509:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776873.565670:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776873.565673:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776873.565676:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776873.691952:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776873.691958:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776873.691959:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776873.691963:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776873.691965:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776873.691967:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776873.691969:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776873.691970:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776873.691972:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776873.819960:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776873.819970:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776873.819973:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776873.819977:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776873.819981:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776873.819988:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18589 00000400:00000200:0.0:1775776873.819997:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776873.820004:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776873.820007:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776873.820010:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820013:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776873.820016:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820018:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776873.820023:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776873.820024:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776873.820027:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776873.820029:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776873.820032:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776873.820034:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776873.820037:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820039:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776873.820040:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820042:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776873.820045:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776873.820048:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820051:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776873.820055:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776873.820058:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776873.820061:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820063:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776873.820065:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776873.820075:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776873.820079:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776873.820082:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776873.820083:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820085:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776873.820087:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776873.820090:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776873.820096:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776873.820101:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776873.820107:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776873.820115:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776873.820118:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776873.820121:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776873.820138:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776873.820141:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776873.820153:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776873.820154:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776873.820156:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776873.820159:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776873.820161:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18589 00000400:00000200:0.0:1775776873.820165:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776873.820168:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776873.820170:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776873.820172:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776873.820173:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776873.820174:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776873.820176:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776873.820177:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776873.820178:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776873.820179:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775776873.820180:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776873.820181:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776873.820183:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776873.820185:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000100:00000040:2.0:1775776873.820185:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776873.820187:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776873.820189:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776873.820192:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776873.820192:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776873.820193:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776873.820194:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776873.820195:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820197:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:2.0:1775776873.820197:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1775776873.820199:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 02000000:00000001:2.0:1775776873.820200:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1775776873.820201:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 02000000:00000001:2.0:1775776873.820202:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820203:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776873.820205:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1775776873.820205:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099805312:192.168.201.139@tcp:250:kworker.0 02000000:00000001:0.0:1775776873.820209:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776873.820209:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776873.820211:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820212:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775776873.820213:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776873.820215:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776873.820217:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775776873.820217:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1775776873.820219:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820221:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776873.820221:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776873.820223:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776873.820224:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776873.820226:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776873.820228:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776873.820229:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776873.820229:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776873.820231:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776873.820232:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1775776873.820233:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 02000000:00000001:0.0:1775776873.820234:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776873.820237:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000400:00000010:2.0:1775776873.820238:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000001:0.0:1775776873.820239:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776873.820243:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775776873.820245:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099805312, portal 25 00000100:00000001:2.0:1775776873.820247:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776873.820249:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741760640 : -114841967790976 : ffff978d454df480) 00000100:00000001:0.0:1775776873.820250:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776873.820252:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776873.820253:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776873.820255:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.820261:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776873.820263:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776873.820264:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00080000:0.0:1775776873.820266:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1775776873.820266:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00080000:0.0:1775776873.820269:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776873.820270:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:2.0:1775776873.820270:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099805312, offset 0 00000100:00080000:0.0:1775776873.820272:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776873.820274:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776873.820276:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18589 00000400:00000200:2.0:1775776873.820276:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776873.820279:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776873.820281:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000200:2.0:1775776873.820283:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:0.0:1775776873.820284:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776873.820286:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820288:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776873.820288:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000040:0.0:1775776873.820289:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776873.820311:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775776873.820315:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776873.820315:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000020:00000001:0.0:1775776873.820317:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:2.0:1775776873.820318:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1775776873.820319:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776873.820320:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1775776873.820321:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000020:00000040:0.0:1775776873.820322:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776873.820324:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776873.820325:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4522:4522:256:4294967295] 192.168.201.139@tcp LPNI seq info [4522:4522:8:4294967295] 00000020:00000001:0.0:1775776873.820326:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776873.820329:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776873.820330:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776873.820331:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776873.820332:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820334:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776873.820336:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776873.820338:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000200:2.0:1775776873.820340:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000001:0.0:1775776873.820341:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776873.820342:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776873.820344:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776873.820346:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776873.820346:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820348:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776873.820350:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776873.820352:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820354:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776873.820357:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776873.820359:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000010:2.0:1775776873.820359:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 02000000:00000001:0.0:1775776873.820362:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776873.820363:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820365:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.0:1775776873.820365:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776873.820366:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776873.820367:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776873.820369:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 02000000:00000010:0.0:1775776873.820370:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776873.820373:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776873.820375:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00100000:0.0:1775776873.820377:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1775776873.820378:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776873.820382:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820384:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:2.0:1775776873.820385:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000100:00000001:0.0:1775776873.820386:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776873.820389:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776873.820390:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775776873.820395:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776873.820395:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1775776873.820398:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776873.820400:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:2.0:1775776873.820400:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000100:00000001:0.0:1775776873.820402:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1775776873.820402:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00080000:0.0:1775776873.820403:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776873.820405:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776873.820406:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1775776873.820408:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18589 00000100:00000200:2.0:1775776873.820409:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776873.820412:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776873.820414:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776873.820417:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776873.820418:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776873.820419:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820421:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776873.820421:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776873.820423:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820425:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776873.820427:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776873.820428:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000100:00000001:2.0:1775776873.820428:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776873.820430:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776873.820430:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:0.0:1775776873.820431:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776873.820433:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776873.820434:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776873.820435:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776873.820437:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776873.820437:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776873.820439:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776873.820440:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313262837 00000100:00000040:0.0:1775776873.820441:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820442:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776873.820444:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000010:2.0:1775776873.820445:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc00. 00000100:00080000:0.0:1775776873.820447:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820449:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776873.820450:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776873.820451:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1775776873.820452:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776873.820454:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820456:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776873.820457:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313262837 00000008:00100000:0.0:1775776873.820458:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776873.820460:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820462:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776873.820465:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776873.820467:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776873.820469:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776873.820471:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820472:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776873.820474:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776873.820475:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776873.820478:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775776873.820480:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776873.820484:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776873.820490:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776873.820492:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776873.820495:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776873.820501:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776873.820503:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776873.820506:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776873.820507:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776873.820509:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776873.820511:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776873.820513:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18589 00000400:00000200:0.0:1775776873.820516:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776873.820519:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776873.820521:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776873.820524:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820526:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776873.820527:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820529:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776873.820542:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776873.820544:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776873.820545:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776873.820547:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776873.820548:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776873.820551:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776873.820553:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820555:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776873.820556:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820558:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776873.820560:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776873.820562:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776873.820564:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776873.820566:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776873.820568:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776873.820570:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820572:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776873.820574:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776873.820576:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820577:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776873.820581:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776873.820583:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776873.820585:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776873.820586:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776873.820588:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776873.820589:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776873.820590:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776873.820594:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776873.820597:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776873.820601:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776873.820608:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776873.820609:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776873.820613:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776873.820619:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776873.820624:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18599) 00000100:00000010:2.0:1775776873.820735:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bcc0. 00000100:00000001:2.0:1775776873.820739:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776873.820743:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776873.820745:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.820748:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776873.820753:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776873/real 1775776873] req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776873.820762:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776873.820764:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776873.820766:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776873.820769:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776873.820771:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776873.820773:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776873.820777:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776873.820830:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.820836:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776873.820838:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.820840:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.820842:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776873.820844:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.820850:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776873.820853:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.820859:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776873.820861:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.820863:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.820864:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.820866:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776873.820868:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776873.820870:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776873.820874:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776873.820876:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776873.820880:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776873.820886:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099805312:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776873.820890:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.820891:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.820893:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df480 x1862033099805312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.820904:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776873.820906:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776873.820908:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776873.820912:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776873.820913:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776873.820915:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776873.820917:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776873.820919:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:2.0:1775776873.820932:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776873.820934:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776873.820936:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df480. 00000100:00000001:2.0:1775776873.820939:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776873.820940:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.820942:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.820949:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776873.820950:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.820952:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776873.820953:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776873.820955:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776873.820958:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.820964:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776873.820966:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776873.820975:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776873.820977:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776873.820979:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776873.820983:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805440:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776873.820986:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776873.820988:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776873.820989:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776873.820991:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776873.820994:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776873.820996:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776873.821000:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776873.821003:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776873.821006:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776873.821009:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099805440, portal 10 00000100:00000001:2.0:1775776873.821011:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776873.821012:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741753472 : -114841967798144 : ffff978d454dd880) 00000100:00000040:2.0:1775776873.821019:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821025:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776873.821028:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776873.821030:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776873.821032:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099805440, offset 0 00000400:00000200:2.0:1775776873.821036:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776873.821040:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776873.821044:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776873.821048:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776873.821050:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776873.821052:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776873.821055:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776873.821058:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4523:4523:256:4294967295] 192.168.201.139@tcp LPNI seq info [4523:4523:8:4294967295] 00000400:00000200:2.0:1775776873.821062:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776873.821068:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776873.821073:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776873.821076:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776873.821080:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776873.821083:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776873.821087:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776873.821090:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776873.821093:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776873.821095:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776873.821098:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776873.821101:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776873.821103:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776873.821105:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776873.821108:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776873.821111:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821119:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.821122:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821128:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.821129:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776873.821132:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776873.821135:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776873.821138:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776873.821140:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313261573 00000100:00000001:2.0:1775776873.821143:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313262837 00000100:00000001:2.0:1775776873.821145:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313261573 00000100:00000001:2.0:1775776873.821147:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313262837 00000100:00000001:2.0:1775776873.821149:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776873.821151:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.821153:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776873.821157:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776873/real 1775776873] req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776873.821164:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776873.821165:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776873.821167:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776873.821169:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776873.821171:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776873.821173:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776873.821176:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776873.821185:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821197:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776873.821198:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.821200:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.821201:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776873.821204:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821210:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776873.821214:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821220:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776873.821223:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.821224:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.821225:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.821227:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776873.821229:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776873.821231:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776873.821234:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776873.821236:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776873.821239:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776873.821245:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805440:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776873.821249:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776873.821251:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821257:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776873.821258:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776873.821260:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776873.821262:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776873.821263:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776873.821265:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805568:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776873.821268:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776873.821270:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776873.821271:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776873.821273:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776873.821276:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776873.821277:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776873.821280:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776873.821282:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776873.821285:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776873.821288:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099805568, portal 10 00000100:00000001:2.0:1775776873.821302:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776873.821303:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748992 : -114841967802624 : ffff978d454dc700) 00000100:00000040:2.0:1775776873.821309:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821316:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776873.821318:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776873.821321:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776873.821323:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099805568, offset 0 00000400:00000200:2.0:1775776873.821327:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776873.821331:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776873.821334:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776873.821337:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776873.821339:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776873.821341:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776873.821344:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776873.821346:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4524:4524:256:4294967295] 192.168.201.139@tcp LPNI seq info [4524:4524:8:4294967295] 00000400:00000200:2.0:1775776873.821358:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776873.821366:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776873.821370:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776873.821373:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776873.821377:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776873.821379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776873.821383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776873.821385:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776873.821388:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776873.821389:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776873.821392:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776873.821395:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776873.821397:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776873.821399:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776873.821401:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776873.821404:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821410:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.821413:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821418:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.821420:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776873.821422:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776873.821424:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776873.821427:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776873.821429:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313261574 00000100:00000001:2.0:1775776873.821431:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313262837 00000100:00000001:2.0:1775776873.821433:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313261574 00000100:00000001:2.0:1775776873.821435:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313262837 00000100:00000001:2.0:1775776873.821436:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776873.821438:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.821440:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776873.821444:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776873/real 1775776873] req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776873.821452:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776873.821454:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776873.821455:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776873.821457:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776873.821471:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776873.821474:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776873.821477:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776873.821485:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821491:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776873.821493:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.821494:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.821495:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776873.821498:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821504:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776873.821507:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821513:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776873.821515:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.821516:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.821517:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.821518:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776873.821519:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776873.821521:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776873.821524:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776873.821526:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776873.821529:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776873.821537:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805568:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776873.821541:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776873.821544:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.821549:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776873.821551:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776873.821553:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776873.821554:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776873.821556:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776873.821559:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805696:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776873.821562:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776873.821563:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776873.821565:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776873.821566:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776873.821569:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776873.821571:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776873.821575:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776873.821577:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776873.822311:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776873.822319:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099805696, portal 4 00000100:00000001:2.0:1775776873.822321:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776873.822322:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741750784 : -114841967800832 : ffff978d454dce00) 00000100:00000040:2.0:1775776873.822330:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.822337:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776873.822339:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776873.822344:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776873.822412:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099805696, offset 0 00000400:00000200:2.0:1775776873.822420:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776873.822425:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776873.822429:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776873.822432:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776873.822434:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776873.822437:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776873.822439:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776873.822442:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4525:4525:256:4294967295] 192.168.201.139@tcp LPNI seq info [4525:4525:8:4294967295] 00000400:00000200:2.0:1775776873.822445:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776873.822451:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776873.822456:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776873.822652:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776873.822658:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776873.822661:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776873.822665:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776873.822667:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776873.822672:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776873.822674:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776873.822677:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776873.822681:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776873.822685:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776873.822686:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776873.822689:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776873.822693:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.822701:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.822704:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.822711:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.822713:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776873.822717:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776873.822719:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776873.822723:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776873.822725:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313261584 00000100:00000001:2.0:1775776873.822727:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313262837 00000100:00000001:2.0:1775776873.822729:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313261584 00000100:00000001:2.0:1775776873.822731:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313262837 00000100:00000001:2.0:1775776873.822733:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776873.822735:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.822737:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776873.822741:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776873/real 1775776873] req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776873.822749:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776873.822750:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776873.822752:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776873.822754:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776873.822756:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776873.822757:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776873.822760:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776873.822767:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.822773:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776873.822774:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.822776:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.822777:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776873.823083:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823090:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776873.823094:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823100:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776873.823104:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823105:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.823106:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823108:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776873.823110:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776873.823112:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776873.823116:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776873.823118:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776873.823121:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776873.823128:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805696:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776873.823133:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776873.823136:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823142:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776873.823144:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776873.823146:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776873.823149:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776873.823151:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776873.823154:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805824:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776873.823157:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776873.823159:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776873.823161:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776873.823163:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776873.823167:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776873.823169:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776873.823173:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776873.823176:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776873.823180:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776873.823184:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099805824, portal 4 00000100:00000001:2.0:1775776873.823186:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776873.823187:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748096 : -114841967803520 : ffff978d454dc380) 00000100:00000040:2.0:1775776873.823193:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823200:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776873.823202:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776873.823205:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776873.823207:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099805824, offset 0 00000400:00000200:2.0:1775776873.823211:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776873.823215:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776873.823219:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776873.823222:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776873.823225:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776873.823227:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776873.823229:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776873.823232:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4526:4526:256:4294967295] 192.168.201.139@tcp LPNI seq info [4526:4526:8:4294967295] 00000400:00000200:2.0:1775776873.823236:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776873.823242:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776873.823246:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776873.823249:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776873.823253:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776873.823256:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1775776873.823260:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776873.823263:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1775776873.823266:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776873.823268:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776873.823271:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776873.823274:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776873.823277:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776873.823279:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776873.823281:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776873.823284:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823310:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.823314:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776898 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823320:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.823321:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776873.823325:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776873.823329:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776873.823332:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776873.823334:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313261576 00000100:00000001:2.0:1775776873.823336:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313262837 00000100:00000001:2.0:1775776873.823337:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313261576 00000100:00000001:2.0:1775776873.823339:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313262837 00000100:00000001:2.0:1775776873.823341:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776873.823343:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.823345:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776873.823348:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776873/real 1775776873] req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776873.823439:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776873.823441:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776873.823442:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776873.823445:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776873.823447:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776873.823449:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776873.823452:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776873.823646:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823655:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776873.823657:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823659:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.823661:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776873.823665:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823672:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776873.823676:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823683:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776873.823685:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823687:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776873.823688:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823690:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776873.823692:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776873.823694:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776873.823697:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776873.823699:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776873.823703:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776873.823711:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099805824:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776873.823715:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823718:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.823721:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd880 x1862033099805440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823727:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776873.823730:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776873.823732:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776873.823736:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776873.823738:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776873.823739:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776873.823742:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776873.823744:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:2.0:1775776873.823748:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776873.823750:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776873.823752:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd880. 00000100:00000001:2.0:1775776873.823755:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776873.823756:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823758:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.823761:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc700 x1862033099805568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823768:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776873.823769:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776873.823771:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776873.823773:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776873.823774:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776873.823774:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776873.823776:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776873.823777:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775776873.823828:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776873.823829:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776873.823831:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc700. 00000100:00000001:2.0:1775776873.823835:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776873.823836:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823838:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.823843:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dce00 x1862033099805696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823850:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776873.823851:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776873.823853:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776873.823856:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776873.823858:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776873.823860:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776873.823862:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776873.823863:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c000. 02000000:00000001:2.0:1775776873.823866:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776873.823867:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776873.823869:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dce00. 00000100:00000001:2.0:1775776873.823872:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776873.823873:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823875:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776873.823879:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc380 x1862033099805824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776898 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776873.823886:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776873.823887:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776873.823888:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776873.823890:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776873.823892:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776873.823893:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776873.823894:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776873.823896:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f400. 02000000:00000001:2.0:1775776873.823898:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776873.823899:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776873.823900:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc380. 00000100:00000001:2.0:1775776873.823902:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776873.823903:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823906:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776873.823914:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776873.823916:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776873.823918:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776873.823919:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776874.524266:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776874.524274:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776874.524276:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776874.524289:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776874.524293:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776874.524297:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776874.524300:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776874.524302:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776874.524305:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776874.587891:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776874.587898:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776874.587901:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776874.587902:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776874.587905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776874.587906:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776874.587908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776874.587909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776874.588082:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776874.588087:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776874.588090:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776874.715897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776874.715905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776874.715906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776874.715912:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776874.715915:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776874.715959:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776874.715974:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776874.715988:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776874.716003:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776875.099937:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313262837, at 4313262848 00000100:00000010:0.0:1775776875.099952:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000040:1.0:1775776875.100040:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313262837, at 4313262848 00000100:00000010:1.0:1775776875.100052:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:3.0:1775776875.547968:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776875.547973:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776875.547975:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776875.547979:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776875.547982:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776875.547988:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776875.547990:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776875.547993:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776875.547994:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776875.613380:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776875.613387:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776875.613393:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776875.613394:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776875.613398:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776875.613399:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776875.613403:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776875.613404:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776875.613500:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776875.613504:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776875.613507:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776875.739967:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776875.739974:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776875.739975:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776875.739982:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776875.739985:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776875.739990:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776875.739992:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776875.739994:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776875.739997:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776876.571901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776876.571908:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776876.571910:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776876.571915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776876.571918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776876.571922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776876.571925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776876.571928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776876.571931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776876.637504:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776876.637511:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776876.637517:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776876.637519:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776876.637522:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776876.637524:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776876.637527:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776876.637528:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776876.637637:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776876.637641:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776876.637643:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776876.763882:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776876.763889:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776876.763891:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776876.763896:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776876.763900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776876.763904:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776876.763906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776876.763909:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776876.763911:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776877.595910:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776877.595923:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776877.595925:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776877.595930:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776877.595934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776877.595938:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776877.595940:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776877.595943:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776877.595945:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776877.660121:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776877.660127:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776877.660132:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776877.660132:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776877.660135:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776877.660136:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776877.660138:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776877.660138:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776877.660187:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776877.660190:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776877.660192:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776877.787859:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776877.787866:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776877.787868:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776877.787873:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776877.787877:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776877.787880:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776877.787883:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776877.787885:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776877.787888:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.619869:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776878.619877:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776878.619879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776878.619884:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776878.619887:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776878.619891:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776878.619893:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776878.619896:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776878.619899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776878.684322:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776878.684329:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776878.684332:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776878.684335:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776878.684339:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776878.685268:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776878.685271:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776878.685273:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776878.685278:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776878.685279:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776878.685283:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776878.685284:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776878.685287:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776878.685288:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776878.686231:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776878.686235:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776878.686238:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776878.812138:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776878.812145:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776878.812148:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776878.812154:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776878.812157:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776878.812161:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776878.812163:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776878.812165:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776878.812168:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776878.947674:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776878.947683:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776878.947686:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776878.947690:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776878.947702:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776878.947710:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18594 00000400:00000200:3.0:1775776878.947727:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776878.947734:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776878.947737:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776878.947740:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947744:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.947746:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947749:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776878.947752:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776878.947754:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776878.947757:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776878.947759:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776878.947762:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776878.947764:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776878.947767:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947769:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.947771:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947774:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776878.947820:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776878.947824:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947826:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776878.947830:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776878.947833:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776878.947836:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.947838:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776878.947840:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776878.947850:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776878.947854:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776878.947857:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776878.947859:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.947862:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776878.947864:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776878.947868:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776878.947875:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776878.947881:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776878.947890:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776878.947900:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776878.947904:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776878.947909:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776878.947936:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776878.947939:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776878.947942:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776878.947944:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776878.947945:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776878.947948:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776878.947950:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18594 00000400:00000200:3.0:1775776878.947954:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776878.947964:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776878.947967:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776878.947970:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947972:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.947974:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947976:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776878.947978:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776878.947979:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776878.947981:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776878.947983:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776878.947985:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776878.947987:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776878.947990:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947991:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.947993:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.947995:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776878.947997:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776878.947999:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948001:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776878.948003:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776878.948008:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776878.948010:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948012:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776878.948015:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776878.948018:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948020:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776878.948023:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776878.948025:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776878.948027:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776878.948029:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948030:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776878.948031:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776878.948033:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776878.948037:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775776878.948040:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776878.948044:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776878.948050:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776878.948052:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776878.948055:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776878.948061:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776878.948064:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776878.948067:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776878.948068:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776878.948070:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776878.948072:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776878.948074:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18594 00000400:00000200:3.0:1775776878.948078:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776878.948080:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776878.948082:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776878.948084:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948086:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.948088:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948090:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776878.948092:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776878.948093:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776878.948094:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776878.948096:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776878.948098:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776878.948101:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776878.948103:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948105:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.948106:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948108:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776878.948110:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776878.948112:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948114:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776878.948115:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776878.948118:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776878.948120:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948122:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776878.948123:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776878.948125:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948127:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776878.948130:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776878.948133:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776878.948135:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776878.948136:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948138:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776878.948140:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776878.948141:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776878.948144:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776878.948147:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776878.948150:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776878.948157:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776878.948159:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776878.948162:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776878.948168:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776878.948171:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776878.948174:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776878.948175:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776878.948176:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776878.948178:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776878.948181:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18594 00000400:00000200:3.0:1775776878.948184:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776878.948186:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776878.948188:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776878.948191:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948193:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.948195:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948197:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776878.948199:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776878.948200:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776878.948228:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776878.948230:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776878.948232:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776878.948234:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776878.948237:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948238:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.948240:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948242:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776878.948243:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776878.948246:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948248:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776878.948250:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776878.948252:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776878.948255:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948257:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776878.948259:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776878.948262:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948263:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776878.948267:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776878.948269:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776878.948271:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776878.948272:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948273:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776878.948275:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776878.948276:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776878.948279:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776878.948283:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776878.948286:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776878.948293:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776878.948294:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776878.948298:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776878.948304:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776878.948306:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776878.948309:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776878.948310:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776878.948311:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776878.948313:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776878.948315:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18594 00000400:00000200:3.0:1775776878.948319:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776878.948322:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776878.948324:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776878.948327:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948329:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.948330:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948332:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776878.948334:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776878.948335:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776878.948336:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776878.948338:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776878.948340:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776878.948342:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776878.948344:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948346:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776878.948347:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948349:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776878.948351:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776878.948353:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776878.948355:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776878.948356:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776878.948359:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776878.948361:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948362:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776878.948364:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776878.948366:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948367:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776878.948370:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776878.948372:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776878.948374:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776878.948375:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776878.948377:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776878.948378:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776878.948379:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776878.948384:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776878.948386:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776878.948390:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776878.948401:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776878.948403:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776878.948407:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776878.948413:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776878.948418:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18604) 00000100:00000001:2.0:1775776878.948557:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776878.948561:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776878.948564:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776878.948569:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.948577:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776878.948579:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.948582:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776878.948585:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776878.948587:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776878.948590:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099805952:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776878.948594:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776878.948598:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776878.948601:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.948604:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776878.948608:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776878.948610:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776878.948616:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776878.948619:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776878.948626:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776878.948633:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099805952, portal 25 00000100:00000001:2.0:1775776878.948635:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776878.948637:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751253504 : -114841958298112 : ffff978d45dece00) 00000100:00000040:2.0:1775776878.948642:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.948648:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776878.948651:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776878.948653:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776878.948657:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099805952, offset 0 00000400:00000200:2.0:1775776878.948663:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776878.948671:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.948676:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776878.948681:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776878.948686:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776878.948690:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776878.948692:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776878.948697:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4527:4527:256:4294967295] 192.168.201.139@tcp LPNI seq info [4527:4527:8:4294967295] 00000400:00000200:2.0:1775776878.948701:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.948711:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776878.948717:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.948723:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776878.948729:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776878.948733:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776878.948739:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776878.948742:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1775776878.948746:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.948749:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776878.948755:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776878.948759:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776878.948763:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776878.948766:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776878.948770:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776878.948773:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.948800:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.948804:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.948809:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.948811:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776878.948816:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776878.948819:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776878.948821:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313267966 00000100:00000010:2.0:1775776878.948826:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b1e0. 00000100:00000001:2.0:1775776878.948833:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776878.948842:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313267966 00000100:00000010:2.0:1775776878.948845:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc60. 00000100:00000001:2.0:1775776878.948847:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776878.948850:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776878.948852:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.948854:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776878.948858:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776878/real 1775776878] req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776878.948867:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776878.948868:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776878.948870:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776878.948872:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776878.948874:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776878.948876:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776878.948879:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776878.948887:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.948893:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776878.948894:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.948896:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.948898:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776878.948902:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.948909:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776878.948913:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.948920:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776878.948922:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.948924:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.948925:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.948927:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776878.948929:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776878.948931:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776878.948934:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776878.948937:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776878.948941:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776878.948948:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099805952:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776878.948952:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776878.948954:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.948972:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776878.948974:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.948975:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776878.948977:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776878.948978:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776878.948981:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806080:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776878.948983:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776878.948985:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776878.948986:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.948987:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776878.948990:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776878.948992:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776878.948995:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776878.948997:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776878.948999:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776878.949002:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099806080, portal 10 00000100:00000001:2.0:1775776878.949003:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776878.949004:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751265152 : -114841958286464 : ffff978d45defb80) 00000100:00000040:2.0:1775776878.949010:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949015:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776878.949017:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776878.949018:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776878.949020:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099806080, offset 0 00000400:00000200:2.0:1775776878.949023:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776878.949027:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.949030:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776878.949033:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776878.949035:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776878.949037:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776878.949039:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776878.949042:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4528:4528:256:4294967295] 192.168.201.139@tcp LPNI seq info [4528:4528:8:4294967295] 00000400:00000200:2.0:1775776878.949045:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.949050:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776878.949053:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.949056:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776878.949059:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776878.949061:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776878.949064:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776878.949066:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1775776878.949069:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.949071:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776878.949073:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776878.949076:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776878.949078:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776878.949079:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776878.949081:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776878.949084:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949090:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.949092:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949098:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.949099:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776878.949102:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776878.949106:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776878.949109:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776878.949111:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313266701 00000100:00000001:2.0:1775776878.949113:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313267966 00000100:00000001:2.0:1775776878.949116:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313266701 00000100:00000001:2.0:1775776878.949117:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313267966 00000100:00000001:2.0:1775776878.949120:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776878.949121:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.949123:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776878.949127:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776878/real 1775776878] req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776878.949134:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776878.949135:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776878.949137:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776878.949139:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776878.949140:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776878.949142:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776878.949145:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776878.949153:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949159:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776878.949160:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949162:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.949163:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776878.949166:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949173:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776878.949176:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949183:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776878.949185:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949186:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.949188:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949189:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776878.949191:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776878.949193:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776878.949197:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776878.949199:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776878.949212:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776878.949220:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806080:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776878.949224:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776878.949226:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949233:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776878.949234:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.949236:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776878.949237:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776878.949239:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776878.949242:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806208:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776878.949245:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776878.949246:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776878.949248:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.949249:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776878.949252:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776878.949254:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776878.949257:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776878.949260:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776878.949263:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776878.949266:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099806208, portal 10 00000100:00000001:2.0:1775776878.949267:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776878.949269:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257088 : -114841958294528 : ffff978d45dedc00) 00000100:00000040:2.0:1775776878.949274:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949280:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776878.949282:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776878.949284:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776878.949287:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099806208, offset 0 00000400:00000200:2.0:1775776878.949290:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776878.949294:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.949297:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776878.949304:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776878.949307:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776878.949309:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776878.949311:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776878.949314:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4529:4529:256:4294967295] 192.168.201.139@tcp LPNI seq info [4529:4529:8:4294967295] 00000400:00000200:2.0:1775776878.949318:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.949324:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776878.949328:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.949332:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776878.949335:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776878.949338:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776878.949342:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776878.949345:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1775776878.949348:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.949350:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776878.949352:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776878.949355:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776878.949358:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776878.949360:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776878.949363:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776878.949366:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949374:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.949377:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949383:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.949384:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776878.949387:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776878.949390:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776878.949393:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776878.949394:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313266702 00000100:00000001:2.0:1775776878.949397:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313267966 00000100:00000001:2.0:1775776878.949399:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313266702 00000100:00000001:2.0:1775776878.949401:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313267966 00000100:00000001:2.0:1775776878.949402:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776878.949404:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.949406:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776878.949409:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776878/real 1775776878] req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776878.949416:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776878.949418:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776878.949419:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776878.949421:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776878.949423:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776878.949425:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776878.949428:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776878.949436:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949443:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776878.949444:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949446:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.949447:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776878.949450:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949455:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776878.949458:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949464:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776878.949465:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949467:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.949468:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949469:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776878.949470:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776878.949472:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776878.949475:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776878.949477:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776878.949480:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776878.949487:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806208:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776878.949491:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776878.949493:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949499:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776878.949500:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.949502:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776878.949504:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776878.949506:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776878.949508:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806336:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776878.949511:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776878.949512:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776878.949514:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.949515:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776878.949518:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776878.949520:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776878.949524:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776878.949526:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776878.949530:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776878.949532:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099806336, portal 4 00000100:00000001:2.0:1775776878.949534:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776878.949535:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751263360 : -114841958288256 : ffff978d45def480) 00000100:00000040:2.0:1775776878.949541:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949547:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776878.949549:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776878.949551:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776878.949553:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099806336, offset 0 00000400:00000200:2.0:1775776878.949557:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776878.949560:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.949563:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776878.949566:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776878.949568:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776878.949570:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776878.949573:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776878.949575:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4530:4530:256:4294967295] 192.168.201.139@tcp LPNI seq info [4530:4530:8:4294967295] 00000400:00000200:2.0:1775776878.949579:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.949585:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776878.949589:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.949592:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776878.949596:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776878.949598:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776878.949602:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776878.949605:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1775776878.949608:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.949610:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776878.949612:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776878.949615:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776878.949618:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776878.949619:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776878.949621:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776878.949624:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949631:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.949633:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949639:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.949640:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776878.949643:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776878.949646:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776878.949649:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776878.949651:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313266711 00000100:00000001:2.0:1775776878.949653:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313267966 00000100:00000001:2.0:1775776878.949655:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313266711 00000100:00000001:2.0:1775776878.949657:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313267966 00000100:00000001:2.0:1775776878.949659:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776878.949661:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.949663:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776878.949666:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776878/real 1775776878] req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776878.949674:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776878.949675:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776878.949677:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776878.949679:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776878.949680:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776878.949682:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776878.949684:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776878.949692:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949698:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776878.949699:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949700:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.949701:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776878.949704:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949711:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776878.949714:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949720:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776878.949722:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949723:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.949724:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949726:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776878.949728:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776878.949730:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776878.949733:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776878.949735:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776878.949738:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776878.949745:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806336:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776878.949748:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776878.949751:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949757:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776878.949758:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.949760:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776878.949762:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776878.949763:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776878.949766:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806464:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776878.949769:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776878.949770:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776878.949772:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776878.949774:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776878.949777:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776878.949801:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776878.949804:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776878.949807:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776878.949809:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776878.949811:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099806464, portal 4 00000100:00000001:2.0:1775776878.949813:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776878.949814:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751262464 : -114841958289152 : ffff978d45def100) 00000100:00000040:2.0:1775776878.949819:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949825:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776878.949827:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776878.949829:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776878.949831:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099806464, offset 0 00000400:00000200:2.0:1775776878.949835:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776878.949838:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.949841:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776878.949844:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776878.949846:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776878.949848:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776878.949850:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776878.949853:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4531:4531:256:4294967295] 192.168.201.139@tcp LPNI seq info [4531:4531:8:4294967295] 00000400:00000200:2.0:1775776878.949857:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776878.949863:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776878.949867:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.949871:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776878.949874:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776878.949877:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1775776878.949880:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776878.949883:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1775776878.949887:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776878.949888:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776878.949891:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776878.949894:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776878.949896:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776878.949898:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776878.949900:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776878.949903:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949909:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.949912:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776903 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949918:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.949920:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776878.949922:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776878.949924:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776878.949927:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776878.949929:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313266703 00000100:00000001:2.0:1775776878.949931:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313267966 00000100:00000001:2.0:1775776878.949933:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313266703 00000100:00000001:2.0:1775776878.949934:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313267966 00000100:00000001:2.0:1775776878.949936:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776878.949938:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.949939:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776878.949943:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776878/real 1775776878] req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776878.949949:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776878.949950:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776878.949951:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776878.949953:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776878.949955:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776878.949956:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776878.949959:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776878.949966:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949972:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776878.949973:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949974:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.949976:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776878.949978:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949984:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776878.949987:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.949993:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776878.949995:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949997:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776878.949998:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.949999:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776878.950001:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776878.950003:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776878.950006:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776878.950008:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776878.950011:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776878.950018:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806464:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776878.950021:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.950023:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.950025:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dece00 x1862033099805952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.950032:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776878.950035:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776878.950037:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776878.950040:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776878.950042:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776878.950043:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776878.950045:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776878.950047:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158800. 02000000:00000001:2.0:1775776878.950051:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776878.950052:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776878.950054:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dece00. 00000100:00000001:2.0:1775776878.950058:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776878.950059:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.950060:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.950063:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45defb80 x1862033099806080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.950075:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776878.950076:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776878.950077:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776878.950079:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776878.950080:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776878.950081:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776878.950083:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776878.950085:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158000. 02000000:00000001:2.0:1775776878.950086:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776878.950087:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776878.950088:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45defb80. 00000100:00000001:2.0:1775776878.950091:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776878.950093:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.950094:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.950097:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedc00 x1862033099806208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.950103:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776878.950104:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776878.950105:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776878.950107:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776878.950108:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776878.950109:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776878.950111:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776878.950112:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a000. 02000000:00000001:2.0:1775776878.950114:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776878.950115:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776878.950116:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedc00. 00000100:00000001:2.0:1775776878.950118:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776878.950119:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.950120:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.950123:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def480 x1862033099806336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.950129:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776878.950130:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776878.950131:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776878.950133:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776878.950134:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776878.950135:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776878.950137:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776878.950138:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115ac00. 02000000:00000001:2.0:1775776878.950140:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776878.950141:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776878.950142:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def480. 00000100:00000001:2.0:1775776878.950144:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776878.950145:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.950146:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776878.950149:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def100 x1862033099806464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776903 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776878.950155:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776878.950156:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776878.950157:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776878.950159:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776878.950160:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776878.950161:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776878.950163:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776878.950164:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:2.0:1775776878.950166:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776878.950167:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776878.950168:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def100. 00000100:00000001:2.0:1775776878.950170:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776878.950171:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.950173:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776878.950180:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776878.950182:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776878.950184:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776878.950186:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776879.643911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776879.643919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776879.643921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776879.643926:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776879.643931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776879.643935:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776879.643937:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776879.643940:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776879.643942:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776879.707922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776879.707930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776879.707936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776879.707937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776879.707941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776879.707942:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776879.707945:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776879.707946:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776879.708045:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776879.708049:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776879.708052:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776879.837274:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776879.837282:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776879.837284:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776879.837289:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776879.837293:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776879.837297:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776879.837300:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776879.837303:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776879.837306:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776880.219948:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313267966, at 4313267968 00000100:00000010:1.0:1775776880.219963:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000040:0.0:1775776880.220216:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313267966, at 4313267968 00000100:00000010:0.0:1775776880.220223:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000001:3.0:1775776880.667894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776880.667901:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776880.667903:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776880.667907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776880.667911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776880.667914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776880.667917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776880.667919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776880.667922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732012:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776880.732019:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732024:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776880.732028:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776880.732030:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776880.732034:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732036:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776880.732038:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732040:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732042:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732045:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776880.732047:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776880.732048:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776880.732049:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732051:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776880.732052:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732053:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732055:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732058:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776880.732059:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776880.732060:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776880.732061:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732063:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776880.732064:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732066:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776880.732067:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776880.732179:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776880.732183:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776880.732186:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776880.859887:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776880.859895:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776880.859897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776880.859905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776880.859911:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776880.860038:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776880.860044:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776880.860047:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776880.860051:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776881.691903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776881.691910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776881.691912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776881.691917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776881.691921:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776881.691925:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776881.691927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776881.691929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776881.691932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776881.755908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776881.755914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776881.755921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776881.755923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776881.755926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776881.755927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776881.755930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776881.755931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776881.756097:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776881.756101:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776881.756104:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776881.883895:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776881.883904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776881.883905:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776881.883911:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776881.883915:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776881.883918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776881.883921:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776881.883923:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776881.883925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776882.715836:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776882.715844:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776882.715846:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776882.715851:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776882.715855:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776882.715858:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776882.715860:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776882.715863:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776882.715865:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776882.780018:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776882.780022:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776882.780026:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776882.780027:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776882.780029:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776882.780030:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776882.780032:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776882.780033:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776882.780176:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776882.780179:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776882.780182:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776882.908454:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776882.908461:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776882.908463:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776882.908469:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776882.908473:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776882.908477:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776882.908479:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776882.908481:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776882.908484:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776883.739906:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776883.739914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776883.739915:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776883.739920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776883.739922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776883.739926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776883.739927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776883.739929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776883.739931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776883.804037:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776883.804043:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776883.804048:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776883.804049:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776883.804053:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776883.804054:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776883.804057:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776883.804059:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776883.804163:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776883.804167:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776883.804170:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776883.931890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776883.931897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776883.931899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776883.931905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776883.931909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776883.931912:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776883.931915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776883.931917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776883.931920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776884.060662:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776884.060670:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776884.060673:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776884.060676:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776884.060680:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776884.060685:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18599 00000400:00000200:3.0:1775776884.060693:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776884.060698:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776884.060700:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776884.060703:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060705:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.060706:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060709:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776884.060712:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776884.060713:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776884.060715:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776884.060717:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776884.060719:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776884.060720:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776884.060722:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060723:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.060724:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060725:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776884.060726:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776884.060728:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060730:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776884.060732:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776884.060734:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776884.060736:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.060737:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776884.060739:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776884.060746:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775776884.060749:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776884.060752:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776884.060753:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.060755:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776884.060756:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776884.060759:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776884.060765:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776884.060770:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776884.060776:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776884.060797:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776884.060800:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776884.060804:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776884.060827:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776884.060830:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776884.060833:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776884.060834:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776884.060836:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776884.060838:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776884.060841:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18599 00000400:00000200:3.0:1775776884.060845:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776884.060848:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776884.060850:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776884.060852:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060854:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.060856:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060857:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776884.060860:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776884.060861:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776884.060862:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776884.060864:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776884.060866:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776884.060867:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776884.060870:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060872:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.060873:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060874:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776884.060876:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776884.060878:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060881:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776884.060883:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776884.060889:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776884.060890:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.060892:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776884.060896:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776884.060899:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.060901:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776884.060905:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000020:00000040:3.0:1775776884.060907:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776884.060909:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776884.060910:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.060912:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776884.060913:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776884.060915:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776884.060918:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776884.060921:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776884.060925:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776884.060933:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776884.060935:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776884.060938:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776884.060945:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776884.060947:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776884.060951:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776884.060952:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776884.060953:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776884.060955:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776884.060958:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18599 00000400:00000200:3.0:1775776884.060961:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776884.060963:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776884.060965:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776884.060968:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060970:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.060971:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060973:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776884.060975:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776884.060976:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776884.060977:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776884.060979:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776884.060980:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776884.060982:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776884.060984:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060986:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.060988:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060990:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776884.060992:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776884.060994:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776884.060996:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776884.060997:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776884.061000:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776884.061002:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061003:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776884.061005:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776884.061007:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061008:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776884.061012:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775776884.061014:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776884.061017:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776884.061018:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061019:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776884.061020:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776884.061022:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776884.061026:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776884.061029:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776884.061032:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776884.061038:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776884.061040:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776884.061043:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776884.061048:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776884.061050:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776884.061053:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776884.061055:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776884.061056:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776884.061058:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776884.061060:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18599 00000400:00000200:3.0:1775776884.061063:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776884.061066:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776884.061068:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776884.061071:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.061073:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.061075:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.061077:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776884.061079:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776884.061080:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776884.061082:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776884.061084:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776884.061086:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776884.061087:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776884.061089:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776884.061092:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776884.061093:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776884.061096:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776884.061096:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.061100:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.061104:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000040:2.0:1775776884.061105:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776884.061108:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776884.061124:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776884.061130:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776884.061130:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776884.061134:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776884.061135:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776884.061138:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776884.061139:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776884.061143:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776884.061143:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061146:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775776884.061147:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776884.061149:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1775776884.061153:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099806592:192.168.201.139@tcp:250:kworker.0 00000008:00000001:3.0:1775776884.061154:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061159:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.061160:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:3.0:1775776884.061163:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 02000000:00000001:2.0:1775776884.061165:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1775776884.061167:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776884.061168:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776884.061171:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776884.061172:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776884.061174:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061177:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1775776884.061178:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:3.0:1775776884.061178:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776884.061182:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776884.061182:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776884.061190:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000400:00000200:2.0:1775776884.061193:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776884.061196:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776884.061198:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000100:00100000:3.0:1775776884.061203:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775776884.061205:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776884.061213:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099806592, portal 25 00000100:00000001:3.0:1775776884.061214:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776884.061218:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776884.061218:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.061221:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751251712 : -114841958299904 : ffff978d45dec700) 00000100:00000040:3.0:1775776884.061224:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776884.061229:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776884.061245:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.061247:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776884.061249:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776884.061252:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00080000:3.0:1775776884.061256:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000010:2.0:1775776884.061257:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000001:3.0:1775776884.061260:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:2.0:1775776884.061262:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099806592, offset 0 00000100:00080000:3.0:1775776884.061263:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776884.061267:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775776884.061269:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00080000:3.0:1775776884.061273:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18599 00000400:00000200:2.0:1775776884.061277:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.061283:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776884.061288:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776884.061292:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776884.061296:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776884.061299:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776884.061303:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4532:4532:256:4294967295] 192.168.201.139@tcp LPNI seq info [4532:4532:8:4294967295] 00000400:00000200:2.0:1775776884.061307:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776884.061312:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776884.061318:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000200:2.0:1775776884.061322:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00080000:3.0:1775776884.061323:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776884.061329:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776884.061333:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1775776884.061334:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000040:3.0:1775776884.061338:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000010:2.0:1775776884.061341:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000100:00000001:3.0:1775776884.061341:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776884.061345:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776884.061346:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776884.061349:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:2.0:1775776884.061351:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000020:00000040:3.0:1775776884.061352:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000800:00000200:2.0:1775776884.061356:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000001:3.0:1775776884.061356:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776884.061359:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000800:00000200:2.1:1775776884.061365:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:3.0:1775776884.061366:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:2.1:1775776884.061369:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1775776884.061370:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776884.061374:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000100:2.0:1775776884.061379:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00000001:3.0:1775776884.061379:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000800:00000010:2.0:1775776884.061382:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000020:00000040:3.0:1775776884.061383:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000400:00000200:2.0:1775776884.061388:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1775776884.061388:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776884.061393:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776884.061398:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1775776884.061398:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776884.061402:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775776884.061404:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 02000000:00000001:3.0:1775776884.061405:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776884.061407:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:3.0:1775776884.061409:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776884.061412:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:3.0:1775776884.061413:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000200:2.0:1775776884.061418:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775776884.061418:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061421:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776884.061428:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776884.061432:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776884.061437:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776884.061439:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061442:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775776884.061443:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1775776884.061444:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1775776884.061448:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776884.061448:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776884.061454:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:2.0:1775776884.061461:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.061461:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.061464:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776884.061469:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776884.061470:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776884.061476:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776884.061480:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313273078 00000100:00000001:3.0:1775776884.061485:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:2.0:1775776884.061487:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775776884.061488:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776884.061494:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.061496:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776884.061505:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313273078 00000100:00000001:3.0:1775776884.061505:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776884.061510:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bba0. 00000100:00000040:3.0:1775776884.061513:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18609) 00000100:00000001:2.0:1775776884.061515:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776884.061520:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776884.061523:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.061526:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776884.061530:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776884/real 1775776884] req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776884.061540:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776884.061542:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776884.061543:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776884.061546:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776884.061547:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776884.061550:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776884.061553:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776884.061562:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.061569:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776884.061570:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.061573:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.061574:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776884.061578:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.061585:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776884.061589:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.061596:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776884.061599:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.061600:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.061602:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.061604:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776884.061606:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776884.061608:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776884.061612:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776884.061614:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776884.061618:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776884.061626:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099806592:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776884.061630:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776884.061633:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.061640:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776884.061642:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776884.061644:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776884.061646:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776884.061648:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776884.061651:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806720:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776884.061654:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776884.061656:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776884.061657:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776884.061660:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776884.061662:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776884.061665:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776884.061669:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776884.061671:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776884.061675:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776884.061677:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099806720, portal 10 00000100:00000001:2.0:1775776884.061679:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776884.061681:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751261568 : -114841958290048 : ffff978d45deed80) 00000100:00000040:2.0:1775776884.061687:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.061694:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776884.061696:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776884.061698:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776884.061700:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099806720, offset 0 00000400:00000200:2.0:1775776884.061703:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776884.061706:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.061709:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776884.061712:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776884.061714:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776884.061716:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776884.061719:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776884.061722:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4533:4533:256:4294967295] 192.168.201.139@tcp LPNI seq info [4533:4533:8:4294967295] 00000400:00000200:2.0:1775776884.061726:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.061732:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776884.061736:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776884.061738:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776884.061742:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776884.061744:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776884.061749:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776884.061751:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776884.061755:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776884.061757:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776884.061759:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776884.061762:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776884.063172:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776884.063175:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776884.063179:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776884.063186:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063196:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.063199:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063206:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.063208:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776884.063212:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776884.063216:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776884.063220:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776884.063222:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313271815 00000100:00000001:2.0:1775776884.063225:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313273078 00000100:00000001:2.0:1775776884.063227:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313271815 00000100:00000001:2.0:1775776884.063229:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313273078 00000100:00000001:2.0:1775776884.063231:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776884.063233:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.063236:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776884.063239:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776884/real 1775776884] req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776884.063248:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776884.063249:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776884.063251:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776884.063254:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776884.063255:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776884.063258:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776884.063261:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776884.063269:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063275:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776884.063276:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063279:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.063280:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776884.063283:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063290:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776884.063293:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063300:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776884.063302:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063304:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.063305:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063307:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776884.063309:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776884.063311:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776884.063314:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776884.063317:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776884.063320:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776884.063328:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806720:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776884.063332:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776884.063335:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063342:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776884.063343:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776884.063346:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776884.063348:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776884.063349:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776884.063352:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806848:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776884.063355:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776884.063357:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776884.063359:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776884.063361:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776884.063365:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776884.063367:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776884.063370:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776884.063373:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776884.063376:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776884.063379:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099806848, portal 10 00000100:00000001:2.0:1775776884.063381:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776884.063382:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751262464 : -114841958289152 : ffff978d45def100) 00000100:00000040:2.0:1775776884.063387:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063394:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776884.063396:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776884.063398:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776884.063400:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099806848, offset 0 00000400:00000200:2.0:1775776884.063404:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776884.063408:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.063412:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776884.063415:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776884.063418:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776884.063420:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776884.063423:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776884.063426:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4534:4534:256:4294967295] 192.168.201.139@tcp LPNI seq info [4534:4534:8:4294967295] 00000400:00000200:2.0:1775776884.063430:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.063437:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776884.063442:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776884.063445:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776884.063450:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776884.063452:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776884.063457:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776884.063459:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776884.063463:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776884.063465:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776884.063468:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776884.063471:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776884.063474:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776884.063475:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776884.063478:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776884.063481:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063488:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.063491:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063497:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.063499:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776884.063501:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776884.063503:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776884.063506:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776884.063508:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313271816 00000100:00000001:2.0:1775776884.063510:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313273078 00000100:00000001:2.0:1775776884.063512:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313271816 00000100:00000001:2.0:1775776884.063515:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313273078 00000100:00000001:2.0:1775776884.063517:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776884.063519:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.063521:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776884.063524:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776884/real 1775776884] req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776884.063531:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776884.063532:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776884.063534:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776884.063536:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776884.063537:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776884.063539:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776884.063542:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776884.063550:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063556:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776884.063557:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063559:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.063560:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776884.063563:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063569:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776884.063572:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063579:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776884.063581:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063582:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.063583:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063586:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776884.063587:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776884.063589:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776884.063592:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776884.063594:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776884.063598:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776884.063605:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806848:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776884.063609:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063611:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.063614:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec700 x1862033099806592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063620:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776884.063622:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776884.063625:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776884.063627:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776884.063628:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776884.063630:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776884.063632:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776884.063634:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:2.0:1775776884.063637:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776884.063639:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776884.063641:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec700. 00000100:00000001:2.0:1775776884.063643:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776884.063645:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063646:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.063649:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deed80 x1862033099806720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063655:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776884.063656:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776884.063658:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776884.063660:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776884.063661:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776884.063662:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776884.063664:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776884.063666:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:2.0:1775776884.063668:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776884.063669:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776884.063670:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deed80. 00000100:00000001:2.0:1775776884.063673:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776884.063674:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063675:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.063678:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def100 x1862033099806848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063683:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776884.063684:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776884.063686:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776884.063687:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776884.063688:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776884.063689:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776884.063691:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776884.063693:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:2.0:1775776884.063695:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776884.063696:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776884.063697:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def100. 00000100:00000001:2.0:1775776884.063698:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776884.063700:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063701:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063708:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776884.063709:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.063711:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776884.063713:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776884.063714:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776884.063717:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063723:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776884.063725:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776884.063727:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776884.063729:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776884.063730:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776884.063733:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806976:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776884.063736:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776884.063738:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776884.063739:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776884.063741:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776884.063743:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776884.063745:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776884.063748:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776884.063751:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776884.063754:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776884.063756:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099806976, portal 4 00000100:00000001:2.0:1775776884.063758:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776884.063759:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751263360 : -114841958288256 : ffff978d45def480) 00000100:00000040:2.0:1775776884.063764:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063771:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776884.063773:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776884.063775:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776884.063777:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099806976, offset 0 00000400:00000200:2.0:1775776884.063795:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776884.063798:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.063801:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776884.063804:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776884.063806:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776884.063808:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776884.063810:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776884.063814:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4535:4535:256:4294967295] 192.168.201.139@tcp LPNI seq info [4535:4535:8:4294967295] 00000400:00000200:2.0:1775776884.063818:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.063824:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776884.063828:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776884.063831:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776884.063834:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776884.063837:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776884.063841:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776884.063844:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776884.063847:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776884.063849:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776884.063852:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776884.063855:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776884.063857:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776884.063859:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776884.063861:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776884.063864:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063872:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.063875:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063881:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.063883:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776884.063885:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776884.063887:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776884.063890:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776884.063892:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313271826 00000100:00000001:2.0:1775776884.063894:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313273078 00000100:00000001:2.0:1775776884.063897:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313271826 00000100:00000001:2.0:1775776884.063899:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313273078 00000100:00000001:2.0:1775776884.063901:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776884.063903:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.063904:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776884.063907:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776884/real 1775776884] req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776884.063915:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776884.063917:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776884.063918:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776884.063920:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776884.063922:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776884.063924:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776884.063927:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776884.063935:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063941:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776884.063942:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063944:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.063945:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776884.063948:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063954:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776884.063957:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.063963:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776884.063965:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063967:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.063968:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.063969:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776884.063971:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776884.063973:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776884.063976:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776884.063978:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776884.063982:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776884.063989:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099806976:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776884.063992:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776884.063995:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064001:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776884.064003:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776884.064004:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776884.064006:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776884.064008:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776884.064011:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807104:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776884.064014:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776884.064016:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776884.064017:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776884.064019:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776884.064022:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776884.064024:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776884.064026:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776884.064029:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776884.064031:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776884.064034:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099807104, portal 4 00000100:00000001:2.0:1775776884.064035:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776884.064037:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257088 : -114841958294528 : ffff978d45dedc00) 00000100:00000040:2.0:1775776884.064042:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064048:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776884.064050:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776884.064052:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776884.064054:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099807104, offset 0 00000400:00000200:2.0:1775776884.064058:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776884.064061:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.064064:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776884.064066:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776884.064069:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776884.064071:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776884.064073:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776884.064076:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4536:4536:256:4294967295] 192.168.201.139@tcp LPNI seq info [4536:4536:8:4294967295] 00000400:00000200:2.0:1775776884.064080:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776884.064086:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776884.064090:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776884.064093:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776884.064096:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776884.064098:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1775776884.064102:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776884.064105:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1775776884.064108:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776884.064110:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776884.064124:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776884.064128:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776884.064130:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776884.064132:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776884.064134:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776884.064137:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064144:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.064147:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776909 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064154:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.064155:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776884.064157:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776884.064160:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776884.064163:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776884.064165:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313271817 00000100:00000001:2.0:1775776884.064167:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313273078 00000100:00000001:2.0:1775776884.064170:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313271817 00000100:00000001:2.0:1775776884.064172:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313273078 00000100:00000001:2.0:1775776884.064173:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776884.064175:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.064177:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776884.064180:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776884/real 1775776884] req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776884.064187:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776884.064188:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776884.064190:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776884.064192:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776884.064193:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776884.064195:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776884.064198:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776884.064206:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064212:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776884.064213:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.064214:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.064216:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776884.064219:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064225:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776884.064228:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064235:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776884.064237:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.064239:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776884.064240:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.064241:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776884.064243:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776884.064245:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776884.064248:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776884.064250:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776884.064254:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776884.064261:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807104:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776884.064264:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.064266:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.064269:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def480 x1862033099806976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064275:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776884.064276:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776884.064277:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776884.064279:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776884.064280:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776884.064282:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776884.064283:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776884.064285:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:2.0:1775776884.064287:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776884.064288:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776884.064289:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def480. 00000100:00000001:2.0:1775776884.064291:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776884.064293:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.064294:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776884.064297:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedc00 x1862033099807104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776909 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776884.064309:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776884.064310:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776884.064312:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776884.064314:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776884.064314:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776884.064316:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776884.064317:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776884.064318:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:2.0:1775776884.064320:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776884.064321:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776884.064322:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedc00. 00000100:00000001:2.0:1775776884.064324:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776884.064325:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.064326:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776884.064329:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776884.064330:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776884.064332:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776884.064333:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776884.763901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776884.763909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776884.763911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776884.763916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776884.763920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776884.763923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776884.763925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776884.763928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776884.763931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776884.827881:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776884.827887:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776884.827893:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776884.827894:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776884.827897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776884.827899:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776884.827901:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776884.827903:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776884.830165:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776884.830170:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776884.830174:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776884.955891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776884.955899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776884.955900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776884.955905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776884.955909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776884.955913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776884.955916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776884.955918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776884.955921:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776885.339971:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313273078, at 4313273088 00000100:00000010:1.0:1775776885.339984:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000040:0.0:1775776885.343005:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313273078, at 4313273091 00000100:00000010:0.0:1775776885.343019:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775776885.787907:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776885.787916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776885.787918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776885.787924:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776885.787927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776885.787931:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776885.787933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776885.787935:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776885.787938:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776885.851872:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776885.851879:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776885.851884:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776885.851885:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776885.851888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776885.851890:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776885.851893:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776885.851894:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776885.852013:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776885.852017:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776885.852020:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776885.979908:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776885.979916:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776885.979918:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776885.979923:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776885.979927:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776885.979931:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776885.979933:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776885.979936:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776885.979939:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776886.811871:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776886.811878:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776886.811880:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776886.811885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776886.811889:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776886.811892:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776886.811895:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776886.811897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776886.811900:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776886.875910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776886.875918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776886.875925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776886.875926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776886.875930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776886.875931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776886.875935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776886.875936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776886.876086:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776886.876090:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776886.876094:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776887.003913:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776887.003921:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776887.003922:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776887.003928:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776887.003931:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776887.003935:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776887.003938:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776887.003940:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776887.003942:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776887.836114:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776887.836121:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776887.836123:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776887.836129:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776887.836133:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776887.836136:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776887.836138:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776887.836141:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776887.836143:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776887.900027:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776887.900034:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776887.900039:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776887.900041:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776887.900044:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776887.900057:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776887.900061:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776887.900063:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776887.900756:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776887.900761:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776887.900764:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776888.027861:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776888.027869:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776888.027871:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776888.027877:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776888.027881:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776888.027885:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776888.027888:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776888.027890:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776888.027894:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776888.859929:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776888.859936:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776888.859937:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776888.859942:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776888.859946:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776888.859950:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776888.859952:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776888.859955:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776888.859957:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776888.923909:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776888.924006:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776888.924009:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776888.924013:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776888.924018:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776888.924020:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776888.924021:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776888.924024:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776888.924038:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776888.924040:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776888.924044:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776888.924045:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776888.924048:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776888.924049:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776888.924265:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776888.924268:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776888.924270:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776889.051913:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776889.051921:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776889.051922:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776889.051928:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776889.051932:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776889.051935:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776889.051938:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776889.051940:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776889.051942:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776889.179905:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776889.179914:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776889.179918:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776889.179923:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776889.179928:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776889.179935:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18604 00000400:00000200:1.0:1775776889.179947:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776889.179953:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776889.179956:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776889.179959:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.179963:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.179966:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.179970:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776889.179974:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776889.179976:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776889.179980:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776889.179984:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776889.179986:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776889.179989:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776889.179992:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.179994:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.179996:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.179998:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776889.180013:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776889.180017:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180020:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776889.180038:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776889.180041:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776889.180044:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180046:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776889.180049:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776889.180055:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776889.180059:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776889.180062:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776889.180064:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180067:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776889.180070:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776889.180074:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776889.180081:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776889.180099:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776889.180110:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776889.180121:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776889.180126:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776889.180132:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776889.180169:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776889.180172:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776889.180174:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776889.180175:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776889.180177:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776889.180179:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776889.180181:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18604 00000400:00000200:1.0:1775776889.180191:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776889.180196:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:2.0:1775776889.180196:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1775776889.180200:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776889.180201:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1775776889.180205:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776889.180206:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775776889.180210:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.180214:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000040:2.0:1775776889.180215:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776889.180218:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776889.180223:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776889.180226:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776889.180229:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776889.180229:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776889.180232:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775776889.180236:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 02000000:00000001:2.0:1775776889.180237:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1775776889.180240:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 02000000:00000001:2.0:1775776889.180241:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1775776889.180243:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 02000000:00000001:2.0:1775776889.180246:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776889.180250:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180254:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1775776889.180254:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099807232:192.168.201.139@tcp:250:kworker.0 00000100:00000040:1.0:1775776889.180260:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776889.180262:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1775776889.180263:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776889.180267:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776889.180269:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 02000000:00000001:2.0:1775776889.180269:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1775776889.180273:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.180273:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776889.180275:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776889.180278:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775776889.180281:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776889.180283:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180284:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:2.0:1775776889.180284:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776889.180288:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775776889.180289:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776889.180292:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180293:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776889.180297:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000010:1.0:1775776889.180299:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776889.180301:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 00000400:00000010:2.0:1775776889.180302:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 02000000:00000001:1.0:1775776889.180303:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776889.180305:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180307:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776889.180308:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776889.180309:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776889.180311:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 02000000:00000010:1.0:1775776889.180313:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775776889.180317:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776889.180319:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099807232, portal 25 00000100:00100000:1.0:1775776889.180320:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776889.180325:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776889.180328:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601024 : -114841421950592 : ffff978d65d6d180) 00000100:00000001:1.0:1775776889.180329:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776889.180332:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776889.180336:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776889.180339:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776889.180345:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776889.180349:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776889.180357:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776889.180358:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775776889.180359:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776889.180362:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776889.180364:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:2.0:1775776889.180366:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00080000:1.0:1775776889.180368:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18604 00000400:00000010:2.0:1775776889.180371:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000400:00000200:1.0:1775776889.180372:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776889.180376:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776889.180378:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1775776889.180378:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099807232, offset 0 00000100:00000040:1.0:1775776889.180382:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180384:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.180386:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180388:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776889.180391:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000020:00000001:1.0:1775776889.180392:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776889.180394:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776889.180396:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776889.180397:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776889.180398:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776889.180400:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776889.180404:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:1.0:1775776889.180406:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180408:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.180409:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180411:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776889.180413:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:2.0:1775776889.180413:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00080000:1.0:1775776889.180418:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180420:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776889.180421:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775776889.180422:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1775776889.180423:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776889.180425:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180427:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775776889.180428:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000008:00100000:1.0:1775776889.180429:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776889.180432:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180433:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776889.180435:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000010:1.0:1775776889.180436:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776889.180439:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000400:00000200:2.0:1775776889.180440:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 02000000:00000001:1.0:1775776889.180441:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776889.180444:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180445:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776889.180447:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776889.180449:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776889.180450:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4537:4537:256:4294967295] 192.168.201.139@tcp LPNI seq info [4537:4537:8:4294967295] 02000000:00000010:1.0:1775776889.180452:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776889.180456:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776889.180459:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00100000:1.0:1775776889.180461:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776889.180472:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776889.180481:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180483:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776889.180487:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776889.180488:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000100:00000040:1.0:1775776889.180493:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1775776889.180501:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776889.180506:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1775776889.180508:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776889.180513:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1775776889.180518:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:1.0:1775776889.180520:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776889.180523:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.1:1775776889.180524:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000100:00080000:1.0:1775776889.180525:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776889.180528:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776889.180532:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18604 00000800:00000100:2.0:1775776889.180536:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000400:00000200:1.0:1775776889.180539:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000800:00000010:2.0:1775776889.180541:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:1.0:1775776889.180542:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776889.180545:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775776889.180547:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1775776889.180548:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180553:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.180557:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776889.180558:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775776889.180562:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776889.180565:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776889.180566:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776889.180568:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776889.180570:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776889.180572:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000400:00000200:2.0:1775776889.180573:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000020:00000001:1.0:1775776889.180574:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000400:00000010:2.0:1775776889.180577:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776889.180583:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776889.180588:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776889.180590:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180595:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.180600:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776889.180602:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775776889.180603:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776889.180607:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000040:2.0:1775776889.180608:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776889.180613:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180619:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776889.180622:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776889.180625:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776889.180626:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776889.180630:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.180631:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776889.180634:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776889.180637:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1775776889.180638:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1775776889.180639:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776889.180643:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776889.180647:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000400:00000200:2.0:1775776889.180647:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776889.180650:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313278197 00000020:00000040:1.0:1775776889.180651:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776889.180656:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776889.180658:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776889.180658:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b7e0. 00000100:00000001:1.0:1775776889.180660:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776889.180662:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776889.180677:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.180677:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 02000000:00000010:1.0:1775776889.180685:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776889.180688:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776889.180694:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776889.180698:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313278197 00000100:00000001:1.0:1775776889.180703:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:2.0:1775776889.180704:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b5a0. 00000100:00000001:1.0:1775776889.180706:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.180710:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000040:1.0:1775776889.180712:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.180715:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776889.180721:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.180725:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1775776889.180730:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776889.180733:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775776889.180734:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776889/real 1775776889] req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776889.180737:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000001:2.0:1775776889.180745:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776889.180747:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1775776889.180749:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1775776889.180750:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000100:00080000:1.0:1775776889.180753:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775776889.180755:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000100:00000001:1.0:1775776889.180758:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776889.180763:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18604 00000400:00000010:2.0:1775776889.180763:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776889.180769:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1775776889.180770:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000200:2.0:1775776889.180774:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776889.180777:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000200:2.0:1775776889.180807:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.180813:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776889.180816:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775776889.180817:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776889.180821:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1775776889.180824:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776889.180825:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776889.180828:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.180833:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00100000:2.0:1775776889.180833:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776889.180838:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776889.180845:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776889.180847:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776889.180849:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776889.180850:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1775776889.180851:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776889.180854:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776889.180856:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776889.180858:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776889.180861:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180864:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776889.180867:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180870:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776889.180873:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776889.180877:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776889.180880:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.180880:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1775776889.180882:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776889.180885:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776889.180886:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775776889.180889:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.180890:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776889.180891:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776889.180892:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1775776889.180894:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776889.180896:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.180896:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776889.180899:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.180900:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:1.0:1775776889.180903:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000100:00080000:2.0:1775776889.180904:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:1.0:1775776889.180906:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776889.180909:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776889.180910:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776889.180911:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:1.0:1775776889.180912:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776889.180914:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775776889.180915:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:1.0:1775776889.180916:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776889.180921:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000040:2.0:1775776889.180923:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776889.180924:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776889.180931:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775776889.180941:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099807232:192.168.201.139@tcp:250:kworker.0 00000100:00000001:1.0:1775776889.180942:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776889.180945:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776889.180950:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.180951:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.180956:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775776889.180960:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776889.180963:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d180 x1862033099807232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776889.180964:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18614) 00000100:00000001:2.0:1775776889.180972:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776889.180974:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776889.180977:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776889.180980:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776889.180981:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776889.180983:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776889.180985:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776889.180988:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3bc00. 02000000:00000001:2.0:1775776889.180991:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776889.180993:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776889.180994:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d180. 00000100:00000001:2.0:1775776889.180997:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776889.180998:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181000:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181007:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776889.181009:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.181011:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776889.181012:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776889.181013:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776889.181017:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181036:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776889.181038:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.181040:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776889.181042:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776889.181044:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776889.181047:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807360:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776889.181051:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776889.181053:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776889.181054:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.181057:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776889.181060:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776889.181062:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776889.181067:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776889.181070:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776889.181075:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776889.181078:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099807360, portal 10 00000100:00000001:2.0:1775776889.181080:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776889.181081:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287608192 : -114841421943424 : ffff978d65d6ed80) 00000100:00000040:2.0:1775776889.181087:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181093:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776889.181095:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776889.181097:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776889.181099:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099807360, offset 0 00000400:00000200:2.0:1775776889.181102:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776889.181106:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776889.181120:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776889.181123:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776889.181126:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776889.181128:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776889.181131:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776889.181134:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4538:4538:256:4294967295] 192.168.201.139@tcp LPNI seq info [4538:4538:8:4294967295] 00000400:00000200:2.0:1775776889.181139:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776889.181145:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776889.181149:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776889.181153:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776889.181156:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776889.181159:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776889.181163:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776889.181165:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776889.181168:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776889.181170:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776889.181173:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776889.181176:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776889.181179:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776889.181181:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776889.181183:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776889.181186:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181194:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776889.181197:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181203:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.181204:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776889.181207:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776889.181210:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776889.181213:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776889.181215:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313276933 00000100:00000001:2.0:1775776889.181218:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313278197 00000100:00000001:2.0:1775776889.181220:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313276933 00000100:00000001:2.0:1775776889.181222:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313278197 00000100:00000001:2.0:1775776889.181224:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776889.181226:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.181227:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776889.181230:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776889/real 1775776889] req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776889.181238:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776889.181239:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776889.181241:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776889.181243:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776889.181244:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776889.181246:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776889.181250:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776889.181257:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181263:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776889.181264:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181266:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776889.181267:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776889.181270:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181276:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776889.181279:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181285:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776889.181286:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181288:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776889.181289:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181290:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776889.181292:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776889.181294:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776889.181297:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776889.181298:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776889.181302:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776889.181309:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807360:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776889.181313:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776889.181315:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181325:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776889.181326:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.181328:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776889.181329:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776889.181331:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776889.181333:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807488:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776889.181336:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776889.181337:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776889.181339:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.181340:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776889.181343:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776889.181345:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776889.181348:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776889.181350:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776889.181352:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776889.181355:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099807488, portal 10 00000100:00000001:2.0:1775776889.181356:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776889.181358:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287600128 : -114841421951488 : ffff978d65d6ce00) 00000100:00000040:2.0:1775776889.181362:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181368:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776889.181370:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776889.181371:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776889.181373:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099807488, offset 0 00000400:00000200:2.0:1775776889.181377:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776889.181380:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776889.181383:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776889.181385:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776889.181388:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776889.181390:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776889.181392:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776889.181395:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4539:4539:256:4294967295] 192.168.201.139@tcp LPNI seq info [4539:4539:8:4294967295] 00000400:00000200:2.0:1775776889.181398:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776889.181404:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776889.181409:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776889.181412:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776889.181415:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776889.181418:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776889.181421:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776889.181424:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776889.181427:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776889.181428:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776889.181431:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776889.181434:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776889.181437:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776889.181438:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776889.181440:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776889.181444:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181451:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776889.181454:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181460:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.181461:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776889.181464:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776889.181466:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776889.181469:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776889.181471:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313276934 00000100:00000001:2.0:1775776889.181473:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313278197 00000100:00000001:2.0:1775776889.181475:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313276934 00000100:00000001:2.0:1775776889.181477:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313278197 00000100:00000001:2.0:1775776889.181479:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776889.181481:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.181483:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776889.181486:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776889/real 1775776889] req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776889.181494:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776889.181495:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776889.181497:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776889.181499:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776889.181501:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776889.181502:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776889.181505:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776889.181512:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181519:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776889.181520:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181522:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776889.181523:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776889.181526:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181532:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776889.181535:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181540:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776889.181543:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181544:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776889.181545:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181547:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776889.181548:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776889.181551:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776889.181554:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776889.181556:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776889.181560:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776889.181567:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807488:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776889.181571:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776889.181574:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181579:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776889.181581:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.181583:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776889.181585:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776889.181587:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776889.181589:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807616:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776889.181592:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776889.181594:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776889.181595:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.181597:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776889.181600:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776889.181601:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776889.181605:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776889.181607:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776889.181610:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776889.181613:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099807616, portal 4 00000100:00000001:2.0:1775776889.181615:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776889.181616:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287596544 : -114841421955072 : ffff978d65d6c000) 00000100:00000040:2.0:1775776889.181621:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181626:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776889.181629:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776889.181631:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776889.181634:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099807616, offset 0 00000400:00000200:2.0:1775776889.181637:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776889.181641:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776889.181644:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776889.181646:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776889.181648:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776889.181650:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776889.181652:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776889.181655:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4540:4540:256:4294967295] 192.168.201.139@tcp LPNI seq info [4540:4540:8:4294967295] 00000400:00000200:2.0:1775776889.181658:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776889.181662:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776889.181665:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776889.181668:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776889.181671:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776889.181673:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776889.181677:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776889.181679:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776889.181683:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776889.181684:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776889.181687:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776889.181690:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776889.181693:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776889.181695:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776889.181697:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776889.181700:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181707:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776889.181710:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181716:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.181717:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776889.181720:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776889.181723:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776889.181726:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776889.181728:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313276943 00000100:00000001:2.0:1775776889.181730:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313278197 00000100:00000001:2.0:1775776889.181732:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313276943 00000100:00000001:2.0:1775776889.181734:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313278197 00000100:00000001:2.0:1775776889.181736:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776889.181738:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.181740:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776889.181743:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776889/real 1775776889] req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776889.181750:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776889.181751:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776889.181752:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776889.181754:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776889.181755:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776889.181757:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776889.181759:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776889.181766:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181772:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776889.181773:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181775:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776889.181776:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776889.181802:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181809:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776889.181813:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181820:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776889.181822:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181823:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776889.181825:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.181826:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776889.181827:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776889.181830:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776889.181833:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776889.181835:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776889.181838:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776889.181845:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807616:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776889.181849:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776889.181851:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.181857:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776889.181858:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.181860:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776889.181862:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776889.181864:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776889.181866:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807744:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776889.182055:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776889.182058:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776889.182060:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776889.182061:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776889.182065:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776889.182067:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776889.182070:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776889.182073:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776889.182076:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776889.182079:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099807744, portal 4 00000100:00000001:2.0:1775776889.182081:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776889.182082:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609984 : -114841421941632 : ffff978d65d6f480) 00000100:00000040:2.0:1775776889.182088:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182094:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776889.182095:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776889.182097:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776889.182099:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099807744, offset 0 00000400:00000200:2.0:1775776889.182101:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776889.182104:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776889.182107:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776889.182109:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776889.182110:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776889.182112:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776889.182114:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776889.182117:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4541:4541:256:4294967295] 192.168.201.139@tcp LPNI seq info [4541:4541:8:4294967295] 00000400:00000200:2.0:1775776889.182120:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776889.182126:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776889.182130:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776889.182133:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776889.182136:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776889.182138:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775776889.182142:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776889.182144:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775776889.182147:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776889.182149:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776889.182152:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776889.182155:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776889.182158:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776889.182159:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776889.182162:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776889.182165:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182173:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776889.182176:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182182:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.182183:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776889.182186:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776889.182188:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776889.182190:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776889.182192:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313276935 00000100:00000001:2.0:1775776889.182194:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313278197 00000100:00000001:2.0:1775776889.182196:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313276935 00000100:00000001:2.0:1775776889.182197:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313278197 00000100:00000001:2.0:1775776889.182198:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776889.182199:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.182200:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776889.182203:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776889/real 1775776889] req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776889.182207:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776889.182208:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776889.182209:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776889.182211:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776889.182212:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776889.182213:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776889.182215:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776889.182219:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182226:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776889.182227:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182228:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776889.182229:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776889.182231:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182234:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776889.182236:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182239:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776889.182241:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182241:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776889.182242:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182243:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776889.182244:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776889.182245:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776889.182247:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776889.182248:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776889.182250:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776889.182254:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099807744:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776889.182256:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182257:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776889.182259:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ed80 x1862033099807360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182263:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776889.182272:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776889.182274:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776889.182276:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776889.182277:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776889.182278:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776889.182279:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776889.182280:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b800. 02000000:00000001:2.0:1775776889.182282:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776889.182283:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776889.182284:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ed80. 00000100:00000001:2.0:1775776889.182286:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776889.182287:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182288:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776889.182290:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ce00 x1862033099807488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182294:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776889.182294:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776889.182295:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776889.182296:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776889.182297:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776889.182297:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776889.182298:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776889.182299:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a400. 02000000:00000001:2.0:1775776889.182300:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776889.182301:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776889.182302:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ce00. 00000100:00000001:2.0:1775776889.182303:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776889.182304:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182304:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776889.182306:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c000 x1862033099807616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182309:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776889.182310:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776889.182310:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776889.182311:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776889.182312:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776889.182312:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776889.182314:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776889.182315:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39400. 02000000:00000001:2.0:1775776889.182316:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776889.182317:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776889.182317:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c000. 00000100:00000001:2.0:1775776889.182318:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776889.182319:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182320:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776889.182321:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f480 x1862033099807744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776889.182324:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776889.182325:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776889.182326:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776889.182326:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776889.182327:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776889.182328:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776889.182328:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776889.182329:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38c00. 02000000:00000001:2.0:1775776889.182330:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776889.182331:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776889.182331:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f480. 00000100:00000001:2.0:1775776889.182332:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776889.182333:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182334:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776889.182338:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776889.182339:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776889.182340:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776889.182341:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776889.883859:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776889.883865:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776889.883867:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776889.883872:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776889.883876:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776889.883880:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776889.883883:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776889.883885:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776889.883888:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776889.948144:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776889.948149:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776889.948153:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776889.948154:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776889.948156:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776889.948157:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776889.948159:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776889.948160:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776889.948257:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776889.948259:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776889.948261:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776890.075885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776890.075892:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776890.075893:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776890.075899:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776890.075902:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776890.075906:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776890.075908:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776890.075910:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776890.075913:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776890.460113:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313278197, at 4313278208 00000100:00000010:1.0:1775776890.460128:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000040:0.0:1775776890.460561:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313278197, at 4313278208 00000100:00000010:0.0:1775776890.460568:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:3.0:1775776890.907886:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776890.907893:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776890.907895:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776890.907900:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776890.907903:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776890.907907:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776890.907909:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776890.907912:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776890.907915:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977260:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776890.977266:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977273:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776890.977276:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776890.977279:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776890.977283:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977286:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776890.977288:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977290:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977292:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977295:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776890.977296:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776890.977298:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776890.977299:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977300:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776890.977301:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977302:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977304:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977307:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776890.977308:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776890.977309:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776890.977311:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977312:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776890.977313:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977315:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776890.977316:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776890.977426:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776890.977430:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776890.977432:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776891.099939:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776891.099946:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776891.099948:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776891.099953:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776891.099957:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776891.099960:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776891.099964:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776891.099966:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776891.099968:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776891.931884:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776891.931891:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776891.931893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776891.931898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776891.931901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776891.931905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776891.931908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776891.931911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776891.931914:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776891.995910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776891.995917:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776891.995922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776891.995923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776891.995927:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776891.995928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776891.995932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776891.995933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776891.996015:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776891.996019:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776891.996022:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776892.123940:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776892.123947:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776892.123948:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776892.123954:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776892.123957:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776892.123961:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776892.123964:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776892.123966:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776892.123968:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776892.955903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776892.955910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776892.955912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776892.955918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776892.955922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776892.955925:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776892.955928:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776892.955930:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776892.955932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776893.020013:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776893.020100:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776893.020106:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776893.020107:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776893.020111:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776893.020112:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776893.020115:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776893.020117:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776893.020229:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776893.020233:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776893.020236:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776893.148031:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776893.148038:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776893.148040:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776893.148045:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776893.148048:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776893.148052:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776893.148055:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776893.148057:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776893.148069:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776893.981376:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776893.981383:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776893.981384:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776893.981390:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776893.981394:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776893.981399:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776893.981401:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776893.981404:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776893.981407:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776894.044139:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776894.044146:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776894.044152:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776894.044153:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776894.044157:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776894.044158:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776894.044161:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776894.044162:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776894.044319:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776894.044322:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776894.044325:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776894.171881:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776894.171887:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776894.171889:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776894.171893:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776894.171897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776894.171900:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776894.171903:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776894.171905:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776894.171908:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776894.300011:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776894.300023:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776894.300026:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776894.300030:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776894.300034:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776894.300041:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18609 00000400:00000200:2.0:1775776894.300051:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776894.300057:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776894.300059:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776894.300062:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300064:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300066:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300067:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776894.300070:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776894.300072:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776894.300073:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776894.300075:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776894.300077:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776894.300078:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776894.300081:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300082:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300084:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300087:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776894.300089:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776894.300092:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300095:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776894.300097:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776894.300101:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776894.300104:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300106:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776894.300108:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776894.300117:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c700. 00000020:00000040:2.0:1775776894.300120:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776894.300124:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776894.300125:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300128:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776894.300131:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776894.300135:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776894.300142:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:2.0:1775776894.300149:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776894.300158:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776894.300169:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776894.300173:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776894.300178:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbf2ea94] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776894.300204:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776894.300208:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776894.300211:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776894.300213:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776894.300214:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776894.300217:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776894.300219:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18609 00000400:00000200:2.0:1775776894.300223:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776894.300226:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776894.300228:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776894.300231:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300233:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300235:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300237:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776894.300239:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776894.300240:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776894.300242:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776894.300244:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776894.300245:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776894.300247:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776894.300250:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300252:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300253:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300255:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776894.300256:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776894.300258:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300259:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776894.300260:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776894.300264:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776894.300265:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300266:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776894.300269:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776894.300271:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300272:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776894.300274:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ed80. 00000020:00000040:2.0:1775776894.300276:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776894.300277:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776894.300279:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300280:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776894.300281:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776894.300282:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776894.300285:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505400. 00000100:00000001:2.0:1775776894.300287:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776894.300290:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776894.300295:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776894.300297:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776894.300299:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006375da9b] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776894.300304:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776894.300306:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776894.300308:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776894.300309:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776894.300310:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776894.300312:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776894.300313:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18609 00000400:00000200:2.0:1775776894.300315:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776894.300317:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776894.300318:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776894.300320:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300321:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300322:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300323:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776894.300325:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776894.300326:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776894.300326:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776894.300327:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776894.300328:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776894.300330:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776894.300332:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300333:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300334:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300335:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776894.300337:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776894.300338:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300339:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776894.300340:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776894.300341:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776894.300342:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300343:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776894.300344:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776894.300345:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300346:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776894.300348:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f800. 00000020:00000040:2.0:1775776894.300349:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776894.300350:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776894.300351:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300351:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776894.300352:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776894.300353:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776894.300355:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504c00. 00000100:00000001:2.0:1775776894.300356:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776894.300358:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776894.300362:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776894.300363:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776894.300365:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8818441] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776894.300369:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776894.300371:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776894.300373:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776894.300374:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776894.300375:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776894.300377:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776894.300378:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18609 00000400:00000200:2.0:1775776894.300379:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776894.300381:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776894.300382:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776894.300383:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300384:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300385:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300386:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776894.300387:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776894.300388:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776894.300389:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776894.300389:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776894.300390:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776894.300391:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776894.300393:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300394:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300395:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300396:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776894.300397:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776894.300398:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300399:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776894.300400:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776894.300401:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776894.300402:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300403:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776894.300405:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776894.300406:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300407:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776894.300409:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c380. 00000020:00000040:2.0:1775776894.300411:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776894.300412:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776894.300412:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300413:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776894.300414:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776894.300414:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776894.300416:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775776894.300418:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776894.300419:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776894.300423:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776894.300424:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776894.300426:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f231afe] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776894.300429:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776894.300431:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776894.300433:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776894.300434:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776894.300435:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776894.300437:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776894.300438:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18609 00000400:00000200:2.0:1775776894.300440:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776894.300441:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776894.300442:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776894.300443:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300444:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300445:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300447:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776894.300448:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776894.300449:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776894.300450:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776894.300451:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776894.300452:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776894.300453:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776894.300455:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300456:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776894.300457:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300458:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776894.300459:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776894.300460:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776894.300461:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776894.300462:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776894.300463:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776894.300464:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300465:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776894.300466:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776894.300467:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300468:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776894.300470:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7df80. 00000020:00000040:2.0:1775776894.300471:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776894.300472:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776894.300472:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776894.300473:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776894.300474:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776894.300474:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776894.300476:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775776894.300478:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776894.300480:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776894.300487:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776894.300488:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776894.300490:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009deec8f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776894.300494:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776894.300497:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18619) 00000100:00000001:0.0:1775776894.301187:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776894.301191:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776894.301193:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776894.301198:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301204:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776894.301205:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.301208:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776894.301209:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776894.301211:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776894.301212:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099807872:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776894.301215:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776894.301218:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776894.301220:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.301222:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776894.301226:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776894.301227:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776894.301232:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776894.301234:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776894.301238:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776894.301248:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099807872, portal 25 00000100:00000001:0.0:1775776894.301249:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776894.301250:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288712448 : -114841420839168 : ffff978d65e7c700) 00000100:00000040:0.0:1775776894.301254:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301259:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776894.301261:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776894.301262:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776894.301264:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099807872, offset 0 00000400:00000200:0.0:1775776894.301268:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776894.301274:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.301278:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776894.301281:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776894.301284:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776894.301287:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776894.301289:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776894.301292:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4542:4542:256:4294967295] 192.168.201.139@tcp LPNI seq info [4542:4542:8:4294967295] 00000400:00000200:0.0:1775776894.301295:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.301301:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776894.301305:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.301309:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776894.301313:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776894.301316:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776894.301320:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776894.301322:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776894.301325:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.301326:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776894.301331:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776894.301334:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776894.301337:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776894.301339:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776894.301342:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776894.301344:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301350:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.301352:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301356:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.301357:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776894.301361:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776894.301364:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776894.301367:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313283318 00000100:00000010:0.0:1775776894.301372:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776894.301381:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776894.301387:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313283318 00000100:00000010:0.0:1775776894.301390:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237360. 00000100:00000001:0.0:1775776894.301393:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776894.301395:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776894.301398:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.301400:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776894.301404:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776894/real 1775776894] req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776894.301413:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776894.301414:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776894.301416:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776894.301418:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776894.301420:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776894.301422:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776894.301425:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776894.301433:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301438:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776894.301439:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.301442:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.301443:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776894.301447:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301453:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776894.301457:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301463:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776894.301466:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.301468:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.301469:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.301471:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776894.301473:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776894.301475:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776894.301479:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776894.301481:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776894.301485:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776894.301493:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099807872:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776894.301497:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776894.301500:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301505:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776894.301506:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.301508:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776894.301510:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776894.301511:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776894.301513:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808000:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776894.301516:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776894.301517:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776894.301519:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.301520:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776894.301524:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776894.301526:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776894.301529:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776894.301532:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776894.301535:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776894.301538:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099808000, portal 10 00000100:00000001:0.0:1775776894.301539:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776894.301540:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288722304 : -114841420829312 : ffff978d65e7ed80) 00000100:00000040:0.0:1775776894.301546:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301552:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776894.301554:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776894.301556:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776894.301558:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099808000, offset 0 00000400:00000200:0.0:1775776894.301562:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776894.301580:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.301584:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776894.301587:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776894.301589:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776894.301591:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776894.301593:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776894.301596:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4543:4543:256:4294967295] 192.168.201.139@tcp LPNI seq info [4543:4543:8:4294967295] 00000400:00000200:0.0:1775776894.301609:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.301616:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776894.301620:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.301624:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776894.301627:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776894.301629:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776894.301633:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776894.301635:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776894.301639:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.301640:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776894.301643:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776894.301646:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776894.301648:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776894.301650:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776894.301652:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776894.301655:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301662:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.301665:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301671:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.301672:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776894.301675:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776894.301679:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776894.301682:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776894.301684:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313282053 00000100:00000001:0.0:1775776894.301687:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313283318 00000100:00000001:0.0:1775776894.301689:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313282053 00000100:00000001:0.0:1775776894.301690:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313283318 00000100:00000001:0.0:1775776894.301692:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776894.301694:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.301696:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776894.301699:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776894/real 1775776894] req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776894.301706:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776894.301708:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776894.301709:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776894.301712:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776894.301713:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776894.301715:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776894.301718:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776894.301726:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301732:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776894.301734:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.301736:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.301737:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776894.301740:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301747:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776894.301750:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301758:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776894.301760:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.301761:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.301762:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.301764:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776894.301766:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776894.301768:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776894.301771:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776894.301772:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776894.301776:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776894.301817:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808000:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776894.301820:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776894.301823:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301830:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776894.301831:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.301832:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776894.301834:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776894.301836:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776894.301838:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808128:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776894.301841:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776894.301843:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776894.301844:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.301846:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776894.301849:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776894.301851:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776894.301854:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776894.301857:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776894.301860:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776894.301862:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099808128, portal 10 00000100:00000001:0.0:1775776894.301864:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776894.301865:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724992 : -114841420826624 : ffff978d65e7f800) 00000100:00000040:0.0:1775776894.301870:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301875:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776894.301878:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776894.301880:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776894.301882:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099808128, offset 0 00000400:00000200:0.0:1775776894.301886:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776894.301890:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.301893:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776894.301896:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776894.301898:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776894.301900:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776894.301902:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776894.301905:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4544:4544:256:4294967295] 192.168.201.139@tcp LPNI seq info [4544:4544:8:4294967295] 00000400:00000200:0.0:1775776894.301909:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.301914:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776894.301919:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.301922:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776894.301926:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776894.301929:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776894.301932:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776894.301950:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776894.301954:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.301956:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776894.301959:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776894.301962:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776894.301965:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776894.301967:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776894.301970:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776894.301973:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.301980:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.301983:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302001:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.302002:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776894.302006:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776894.302008:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776894.302012:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776894.302013:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313282055 00000100:00000001:0.0:1775776894.302016:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313283318 00000100:00000001:0.0:1775776894.302018:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313282055 00000100:00000001:0.0:1775776894.302020:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313283318 00000100:00000001:0.0:1775776894.302022:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776894.302023:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.302025:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776894.302029:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776894/real 1775776894] req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776894.302037:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776894.302038:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776894.302040:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776894.302043:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776894.302044:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776894.302046:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776894.302049:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776894.302057:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302064:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776894.302065:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302067:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.302068:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776894.302071:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302087:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776894.302091:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302098:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776894.302100:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302102:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.302103:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302105:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776894.302106:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776894.302109:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776894.302112:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776894.302114:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776894.302118:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776894.302125:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808128:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776894.302129:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776894.302132:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302139:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776894.302140:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.302142:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776894.302144:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776894.302146:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776894.302148:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808256:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776894.302151:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776894.302153:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776894.302154:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.302156:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776894.302159:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775776894.302161:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776894.302164:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776894.302166:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776894.302170:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776894.302173:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099808256, portal 4 00000100:00000001:0.0:1775776894.302174:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776894.302176:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288711552 : -114841420840064 : ffff978d65e7c380) 00000100:00000040:0.0:1775776894.302181:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302187:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776894.302190:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776894.302192:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776894.302194:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099808256, offset 0 00000400:00000200:0.0:1775776894.302198:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776894.302202:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.302206:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776894.302208:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776894.302210:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776894.302213:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776894.302215:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776894.302218:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4545:4545:256:4294967295] 192.168.201.139@tcp LPNI seq info [4545:4545:8:4294967295] 00000400:00000200:0.0:1775776894.302221:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.302227:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776894.302231:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.302234:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776894.302238:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776894.302240:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776894.302244:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776894.302246:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776894.302250:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.302252:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776894.302254:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776894.302257:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776894.302260:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776894.302262:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776894.302265:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776894.302268:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302276:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.302278:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302285:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.302286:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776894.302290:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776894.302293:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776894.302296:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776894.302298:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313282064 00000100:00000001:0.0:1775776894.302300:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313283318 00000100:00000001:0.0:1775776894.302302:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313282064 00000100:00000001:0.0:1775776894.302304:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313283318 00000100:00000001:0.0:1775776894.302306:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776894.302308:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.302309:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776894.302312:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776894/real 1775776894] req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776894.302320:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776894.302321:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776894.302323:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776894.302325:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776894.302327:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776894.302329:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776894.302331:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776894.302339:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302346:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776894.302347:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302349:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.302350:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776894.302353:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302359:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776894.302363:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302369:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776894.302372:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302373:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.302374:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302376:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776894.302378:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776894.302380:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776894.302383:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776894.302385:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776894.302390:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776894.302397:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808256:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776894.302400:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776894.302403:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302409:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776894.302410:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.302412:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776894.302413:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776894.302415:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776894.302418:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808384:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776894.302420:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776894.302422:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776894.302423:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776894.302425:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776894.302428:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759cc00. 02000000:00000001:0.0:1775776894.302430:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776894.302433:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776894.302435:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776894.302438:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776894.302440:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099808384, portal 4 00000100:00000001:0.0:1775776894.302442:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776894.302443:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288718720 : -114841420832896 : ffff978d65e7df80) 00000100:00000040:0.0:1775776894.302449:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302454:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776894.302456:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776894.302459:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776894.302461:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099808384, offset 0 00000400:00000200:0.0:1775776894.302465:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776894.302469:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.302473:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776894.302475:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776894.302478:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776894.302480:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776894.302482:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776894.302484:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4546:4546:256:4294967295] 192.168.201.139@tcp LPNI seq info [4546:4546:8:4294967295] 00000400:00000200:0.0:1775776894.302488:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776894.302494:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776894.302498:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.302501:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776894.302504:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776894.302507:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776894.302510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776894.302512:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776894.302516:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776894.302518:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776894.302520:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776894.302523:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776894.302525:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776894.302527:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776894.302529:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776894.302532:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302540:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.302543:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776919 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302548:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.302549:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776894.302552:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776894.302554:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776894.302557:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776894.302559:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313282055 00000100:00000001:0.0:1775776894.302562:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313283318 00000100:00000001:0.0:1775776894.302564:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313282055 00000100:00000001:0.0:1775776894.302565:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313283318 00000100:00000001:0.0:1775776894.302567:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776894.302569:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.302570:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776894.302573:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776894/real 1775776894] req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776894.302579:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776894.302581:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776894.302582:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776894.302584:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776894.302585:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776894.302587:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776894.302589:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776894.302596:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302602:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776894.302603:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302604:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.302605:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776894.302608:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302614:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776894.302617:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302623:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776894.302625:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302627:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776894.302628:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302630:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776894.302631:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776894.302633:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776894.302636:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776894.302638:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776894.302641:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776894.302649:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808384:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776894.302652:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302654:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.302657:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c700 x1862033099807872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302663:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776894.302666:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776894.302668:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776894.302672:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776894.302674:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776894.302676:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776894.302678:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776894.302680:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:0.0:1775776894.302683:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776894.302685:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776894.302687:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c700. 00000100:00000001:0.0:1775776894.302689:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776894.302691:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302692:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.302695:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ed80 x1862033099808000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302707:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776894.302708:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776894.302709:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776894.302711:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776894.302712:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776894.302713:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776894.302715:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776894.302716:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505400. 02000000:00000001:0.0:1775776894.302718:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776894.302720:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776894.302721:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ed80. 00000100:00000001:0.0:1775776894.302724:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776894.302725:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302727:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.302730:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f800 x1862033099808128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302736:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776894.302737:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776894.302738:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776894.302740:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776894.302741:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776894.302741:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776894.302743:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776894.302744:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:0.0:1775776894.302746:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776894.302747:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776894.302748:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f800. 00000100:00000001:0.0:1775776894.302749:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776894.302750:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302751:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.302754:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099808256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302759:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776894.302760:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776894.302761:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775776894.302763:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776894.302764:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776894.302765:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776894.302766:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776894.302767:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:0.0:1775776894.302769:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776894.302770:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776894.302771:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:0.0:1775776894.302773:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776894.302774:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302775:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776894.302811:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7df80 x1862033099808384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776919 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776894.302818:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776894.302819:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776894.302821:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759cc00. 02000000:00000001:0.0:1775776894.302823:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776894.302824:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776894.302825:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776894.302827:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776894.302828:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:0.0:1775776894.302830:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776894.302831:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776894.302832:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7df80. 00000100:00000001:0.0:1775776894.302834:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776894.302836:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302838:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776894.302844:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776894.302846:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776894.302848:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776894.302850:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776895.003886:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776895.003891:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776895.003893:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776895.003897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776895.003899:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776895.003902:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776895.003904:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776895.003906:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776895.003916:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776895.067902:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776895.067908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776895.067915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776895.067916:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776895.067919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776895.067931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776895.067935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776895.067936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776895.068061:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776895.068064:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776895.068067:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776895.196512:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776895.196518:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776895.196519:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776895.196523:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776895.196526:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776895.196529:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776895.196531:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776895.196533:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776895.196535:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776895.579977:0:314644:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313283318, at 4313283328 00000100:00000010:0.0:1775776895.579994:0:314644:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775776895.580814:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313283318, at 4313283329 00000100:00000010:1.0:1775776895.580819:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237360. 00000100:00000001:3.0:1775776896.027869:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776896.027875:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776896.027877:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776896.027881:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776896.027883:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776896.027886:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776896.027888:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776896.027890:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776896.027891:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776896.091935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776896.091943:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776896.091949:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776896.091951:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776896.091954:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776896.091956:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776896.091960:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776896.091961:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776896.092022:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776896.092026:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776896.092029:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776896.219841:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776896.219847:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776896.219849:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776896.219853:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776896.219856:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776896.219859:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776896.219861:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776896.219879:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776896.219882:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775776896.859920:0:314648:0:(osc_request.c:745:osc_update_next_shrink()) next time 19817 to shrink grant 00000008:00000020:2.0:1775776896.859929:0:314648:0:(osc_request.c:745:osc_update_next_shrink()) next time 19817 to shrink grant 00000100:00000001:3.0:1775776897.051904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776897.051911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776897.051913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776897.051918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776897.051921:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776897.051925:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776897.051928:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776897.051930:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776897.051933:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776897.115917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776897.115923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776897.115929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776897.115930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776897.115933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776897.115934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776897.115937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776897.115939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776897.117932:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776897.117937:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776897.117941:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776897.243926:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776897.243933:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776897.243935:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776897.243940:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776897.243943:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776897.243947:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776897.243950:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776897.243952:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776897.243955:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776898.076059:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776898.076067:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776898.076069:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776898.076075:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776898.076078:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776898.076082:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776898.076085:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776898.076087:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776898.076089:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776898.140487:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776898.140493:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776898.140499:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776898.140501:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776898.140505:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776898.140506:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776898.140510:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776898.140511:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776898.140574:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776898.140578:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776898.140582:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776898.267896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776898.267904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776898.267906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776898.267912:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776898.267916:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776898.267920:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776898.267922:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776898.267925:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776898.267927:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.100147:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776899.100155:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776899.100157:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776899.100163:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776899.100166:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776899.100170:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776899.100172:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776899.100175:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776899.100178:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776899.163920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776899.163930:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776899.163932:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776899.163936:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776899.163941:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776899.163943:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776899.163945:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776899.163948:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776899.163951:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776899.163953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776899.163956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776899.163958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776899.163961:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776899.163962:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776899.164057:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776899.164061:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776899.164064:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776899.292138:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776899.292144:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776899.292146:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776899.292151:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776899.292155:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776899.292158:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776899.292161:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776899.292163:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776899.292166:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776899.419973:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776899.419985:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776899.419989:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776899.419993:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776899.419997:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776899.420005:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18614 00000400:00000200:3.0:1775776899.420015:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776899.420021:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776899.420023:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776899.420026:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420029:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.420032:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420034:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776899.420038:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776899.420040:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776899.420042:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776899.420044:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776899.420046:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776899.420048:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776899.420050:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420052:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.420053:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420055:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776899.420057:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776899.420060:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420062:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776899.420065:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776899.420068:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776899.420071:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.420073:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776899.420076:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776899.420085:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776899.420088:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776899.420091:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776899.420093:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.420096:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776899.420098:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776899.420102:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776899.420109:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776899.420115:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776899.420124:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776899.420134:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776899.420139:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776899.420144:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776899.420169:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776899.420172:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776899.420175:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776899.420176:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776899.420177:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776899.420180:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776899.420182:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18614 00000400:00000200:3.0:1775776899.420186:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776899.420189:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776899.420190:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776899.420193:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420194:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.420196:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420198:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776899.420200:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776899.420201:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776899.420203:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776899.420204:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776899.420206:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776899.420208:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776899.420211:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420213:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.420215:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420217:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776899.420219:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776899.420221:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420225:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776899.420227:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776899.420232:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776899.420234:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.420235:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776899.420240:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776899.420242:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.420244:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776899.420247:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776899.420249:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776899.420251:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776899.420252:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.420254:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776899.420255:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776899.420257:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776899.420260:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775776899.420263:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776899.420268:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776899.420276:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776899.420278:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776899.420281:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776899.420288:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776899.420291:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776899.420294:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776899.420296:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776899.420297:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776899.420299:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776899.420302:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18614 00000400:00000200:3.0:1775776899.420305:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776899.420309:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776899.420311:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776899.420314:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420316:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.420317:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.420319:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776899.420321:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776899.420322:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776899.420323:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776899.420325:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776899.420327:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776899.420328:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776899.420331:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776899.421918:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776899.421924:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776899.421926:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776899.421936:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.421946:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776899.421948:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.422046:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.422049:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422051:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776899.422054:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776899.422058:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422061:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776899.422063:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776899.422068:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776899.422070:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.422073:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776899.422076:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776899.422079:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.422081:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776899.422087:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776899.422090:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776899.422093:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776899.422094:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.422096:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776899.422099:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776899.422102:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776899.422107:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776899.422111:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776899.422117:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776899.422126:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776899.422130:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776899.422134:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776899.422145:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776899.422148:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776899.422152:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776899.422154:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776899.422156:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776899.422159:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776899.422162:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18614 00000400:00000200:3.0:1775776899.422169:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776899.422174:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776899.422177:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776899.422180:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422182:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.422184:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422186:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776899.422188:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776899.422190:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776899.422192:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776899.422194:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776899.422196:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776899.422199:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776899.422202:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422204:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.422206:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422207:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776899.422209:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776899.422211:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422213:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776899.422215:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776899.422218:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776899.422220:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.422221:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776899.422223:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776899.422226:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.422228:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776899.422231:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776899.422233:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776899.422236:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776899.422237:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.422239:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776899.422240:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776899.422242:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776899.422245:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776899.422248:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776899.422252:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776899.422260:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776899.422262:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776899.422265:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776899.422274:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776899.422276:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776899.422280:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776899.422282:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776899.422283:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776899.422286:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776899.422288:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18614 00000400:00000200:3.0:1775776899.422292:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776899.422295:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776899.422297:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776899.422300:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422302:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.422304:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.422305:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:0.0:1775776899.422389:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776899.422392:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776899.422395:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776899.422399:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099808512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776899.422404:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776899.422407:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776899.422410:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.422412:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776899.422425:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776899.422427:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776899.422433:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776899.422436:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776899.422442:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776899.422449:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099808512, portal 25 00000100:00000001:0.0:1775776899.422451:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776899.422452:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751265152 : -114841958286464 : ffff978d45defb80) 00000100:00000040:0.0:1775776899.422457:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422463:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776899.422466:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776899.422468:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776899.422471:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099808512, offset 0 00000400:00000200:0.0:1775776899.422476:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776899.422483:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.422488:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776899.422493:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776899.422496:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776899.422500:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776899.422502:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776899.422505:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4547:4547:256:4294967295] 192.168.201.139@tcp LPNI seq info [4547:4547:8:4294967295] 00000400:00000200:0.0:1775776899.422509:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.422518:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776899.422523:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.422528:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776899.422533:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776899.422536:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776899.422542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776899.422545:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776899.422548:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.422550:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776899.422556:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776899.422560:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776899.422564:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776899.422567:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776899.422570:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776899.422574:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422582:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.422584:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422591:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.422592:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776899.422597:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776899.422600:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776899.422602:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313288439 00000100:00000010:0.0:1775776899.422607:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776899.422613:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776899.422620:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313288439 00000100:00000010:0.0:1775776899.422623:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f2372a0. 00000100:00000001:0.0:1775776899.422625:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776899.422628:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776899.422630:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.422633:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776899.422636:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776899/real 1775776899] req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776899.422644:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776899.422646:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776899.422648:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776899.422650:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776899.422652:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776899.422654:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776899.422658:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776899.422666:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422673:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776899.422675:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.422677:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.422679:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776899.422683:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422690:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776899.422694:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422701:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776899.422704:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.422705:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.422707:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.422709:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776899.422711:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776899.422714:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776899.422718:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776899.422720:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776899.422723:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776899.422730:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099808512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776899.422734:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776899.422737:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422744:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776899.422745:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.422747:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776899.422748:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776899.422750:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776899.422753:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776899.422758:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776899.422761:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776899.422762:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.422764:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776899.422766:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776899.422768:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776899.422771:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776899.422773:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776899.422777:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776899.422802:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099808640, portal 10 00000100:00000001:0.0:1775776899.422804:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776899.422805:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751253504 : -114841958298112 : ffff978d45dece00) 00000100:00000040:0.0:1775776899.422812:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422818:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776899.422819:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776899.422821:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776899.422823:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099808640, offset 0 00000400:00000200:0.0:1775776899.422826:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776899.422830:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.422833:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776899.422835:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776899.422838:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776899.422841:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776899.422870:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776899.422873:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4548:4548:256:4294967295] 192.168.201.139@tcp LPNI seq info [4548:4548:8:4294967295] 00000400:00000200:0.0:1775776899.422878:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.422883:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776899.422888:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.422891:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776899.422895:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776899.422897:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776899.422901:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776899.422903:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776899.422905:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.422907:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776899.422911:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776899.422915:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776899.422917:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776899.422919:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776899.422921:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776899.422924:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422931:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.422934:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.422939:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.422940:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776899.422942:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776899.422945:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776899.422948:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776899.422949:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313287175 00000020:00000001:3.0:1775776899.423100:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776899.423102:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776899.423104:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776899.423105:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776899.423107:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776899.423109:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776899.423113:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.423114:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776899.423116:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776899.423118:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776899.423120:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776899.423123:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776899.423125:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776899.423127:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776899.423129:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776899.423131:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.423133:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776899.423135:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776899.423137:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.423139:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776899.423142:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776899.423144:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776899.423146:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776899.423148:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776899.423150:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776899.423151:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776899.423153:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776899.423158:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776899.423162:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776899.423166:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776899.423173:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776899.423175:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776899.423178:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776899.423185:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776899.423189:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18624) 00000100:00000001:0.0:1775776899.423733:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313288439 00000100:00000001:0.0:1775776899.423736:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313287175 00000100:00000001:0.0:1775776899.423738:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313288439 00000100:00000001:0.0:1775776899.423740:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776899.423742:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.423744:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776899.423749:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776899/real 1775776899] req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776899.423758:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776899.423759:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776899.423761:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776899.423764:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776899.423765:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776899.423767:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776899.423771:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776899.423797:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.423804:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776899.423806:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.423808:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.423809:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776899.423813:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.423819:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776899.423822:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.423828:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776899.423830:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.423832:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.423833:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.423834:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776899.423836:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776899.423838:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776899.423842:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776899.423855:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776899.423859:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776899.423866:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776899.423870:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.423872:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.423875:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45defb80 x1862033099808512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.423881:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776899.423883:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776899.423886:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776899.423888:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776899.423890:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776899.423891:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776899.423893:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776899.423896:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158400. 02000000:00000001:0.0:1775776899.423900:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776899.423903:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776899.423904:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45defb80. 00000100:00000001:0.0:1775776899.423908:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776899.423909:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.423911:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.423914:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dece00 x1862033099808640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.423921:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776899.423922:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776899.423924:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776899.423926:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776899.423927:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776899.423928:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776899.423930:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776899.423932:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:0.0:1775776899.423934:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776899.423935:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776899.423937:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dece00. 00000100:00000001:0.0:1775776899.423939:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776899.423940:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.423942:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.423961:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776899.423963:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.423965:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776899.423967:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776899.423968:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776899.423972:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.423978:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776899.423980:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.423983:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776899.423985:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776899.423987:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776899.423990:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776899.423994:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776899.423996:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776899.423998:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.423999:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776899.424003:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776899.424005:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776899.424009:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776899.424012:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776899.424016:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776899.424020:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099808768, portal 10 00000100:00000001:0.0:1775776899.424022:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776899.424024:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751259776 : -114841958291840 : ffff978d45dee680) 00000100:00000040:0.0:1775776899.424030:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424036:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776899.424039:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776899.424041:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776899.424043:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099808768, offset 0 00000400:00000200:0.0:1775776899.424049:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776899.424054:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.424058:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776899.424061:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776899.424064:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776899.424067:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776899.424069:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776899.424072:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4549:4549:256:4294967295] 192.168.201.139@tcp LPNI seq info [4549:4549:8:4294967295] 00000400:00000200:0.0:1775776899.424076:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.424082:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776899.424087:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.424092:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776899.424096:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776899.424099:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776899.424103:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776899.424106:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776899.424110:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.424111:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776899.424114:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776899.424118:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776899.424121:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776899.424123:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776899.424125:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776899.424128:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424136:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.424138:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424144:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.424145:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776899.424148:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776899.424151:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776899.424154:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776899.424156:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313287177 00000100:00000001:0.0:1775776899.424158:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313288439 00000100:00000001:0.0:1775776899.424160:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313287177 00000100:00000001:0.0:1775776899.424162:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313288439 00000100:00000001:0.0:1775776899.424164:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776899.424166:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.424167:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776899.424171:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776899/real 1775776899] req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776899.424178:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776899.424179:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776899.424181:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776899.424183:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776899.424184:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776899.424186:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776899.424189:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776899.424196:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424202:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776899.424203:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424205:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.424206:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776899.424209:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424215:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776899.424219:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424226:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776899.424228:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424229:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.424230:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424232:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776899.424234:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776899.424236:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776899.424239:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776899.424242:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776899.424245:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776899.424254:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776899.424258:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776899.424261:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424267:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776899.424268:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.424270:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776899.424272:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776899.424274:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776899.424276:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776899.424279:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776899.424281:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776899.424282:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.424284:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776899.424287:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776899.424288:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776899.424291:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776899.424294:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776899.424297:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776899.424300:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099808896, portal 4 00000100:00000001:0.0:1775776899.424302:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776899.424303:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751252608 : -114841958299008 : ffff978d45deca80) 00000100:00000040:0.0:1775776899.424309:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424314:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776899.424316:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776899.424318:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776899.424321:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099808896, offset 0 00000400:00000200:0.0:1775776899.424325:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776899.424328:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.424332:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776899.424334:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776899.424337:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776899.424339:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776899.424341:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776899.424344:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4550:4550:256:4294967295] 192.168.201.139@tcp LPNI seq info [4550:4550:8:4294967295] 00000400:00000200:0.0:1775776899.424348:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.424353:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776899.424357:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.424360:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776899.424363:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776899.424365:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776899.424369:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776899.424371:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776899.424374:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.424376:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776899.424379:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776899.424382:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776899.424385:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776899.424386:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776899.424388:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776899.424391:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424398:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.424401:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424407:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.424408:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776899.424411:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776899.424414:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776899.424416:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776899.424418:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313287186 00000100:00000001:0.0:1775776899.424420:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313288439 00000100:00000001:0.0:1775776899.424422:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313287186 00000100:00000001:0.0:1775776899.424424:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313288439 00000100:00000001:0.0:1775776899.424425:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776899.424427:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.424429:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776899.424432:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776899/real 1775776899] req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776899.424439:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776899.424440:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776899.424442:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776899.424444:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776899.424446:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776899.424447:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776899.424450:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776899.424457:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424462:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776899.424463:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424465:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.424466:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776899.424469:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424476:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776899.424479:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424485:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776899.424487:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424488:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.424489:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424491:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776899.424492:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776899.424494:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776899.424497:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776899.424499:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776899.424502:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776899.424509:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099808896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776899.424513:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776899.424516:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424522:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776899.424523:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.424525:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776899.424527:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776899.424528:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776899.424531:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776899.424533:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776899.424535:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776899.424537:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776899.424538:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776899.424541:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776899.424542:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776899.424545:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776899.424547:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776899.424549:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776899.424551:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099809024, portal 4 00000100:00000001:0.0:1775776899.424553:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776899.424554:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751250816 : -114841958300800 : ffff978d45dec380) 00000100:00000040:0.0:1775776899.424558:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424563:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776899.424565:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776899.424567:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776899.424569:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099809024, offset 0 00000400:00000200:0.0:1775776899.424572:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776899.424575:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.424578:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776899.424580:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776899.424582:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776899.424584:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776899.424586:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776899.424588:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4551:4551:256:4294967295] 192.168.201.139@tcp LPNI seq info [4551:4551:8:4294967295] 00000400:00000200:0.0:1775776899.424592:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776899.424596:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776899.424600:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.424603:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776899.424605:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776899.424607:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776899.424611:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776899.424613:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776899.424616:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776899.424618:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776899.424620:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776899.424623:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776899.424625:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776899.424626:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776899.424628:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776899.424631:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424636:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.424638:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776924 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424648:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.424649:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776899.424652:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776899.424654:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776899.424657:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776899.424659:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313287177 00000100:00000001:0.0:1775776899.424661:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313288439 00000100:00000001:0.0:1775776899.424662:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313287177 00000100:00000001:0.0:1775776899.424664:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313288439 00000100:00000001:0.0:1775776899.424665:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776899.424667:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.424668:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776899.424672:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776899/real 1775776899] req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776899.424678:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776899.424679:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776899.424681:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776899.424683:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776899.424684:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776899.424686:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776899.424689:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776899.424696:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424702:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776899.424703:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424705:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.424706:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776899.424709:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424715:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776899.424717:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424723:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776899.424725:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424726:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776899.424727:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424729:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776899.424730:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776899.424733:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776899.424735:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776899.424737:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776899.424741:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776899.424748:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776899.424752:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424754:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.424756:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee680 x1862033099808768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424762:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776899.424764:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776899.424766:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776899.424768:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776899.424770:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776899.424772:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776899.424773:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776899.424775:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:0.0:1775776899.424777:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776899.424807:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776899.424809:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee680. 00000100:00000001:0.0:1775776899.424813:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776899.424814:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424816:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.424819:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deca80 x1862033099808896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424824:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776899.424825:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776899.424827:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776899.424829:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776899.424830:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776899.424831:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776899.424832:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776899.424833:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158c00. 02000000:00000001:0.0:1775776899.424836:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776899.424837:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776899.424838:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deca80. 00000100:00000001:0.0:1775776899.424839:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776899.424840:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424841:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776899.424856:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec380 x1862033099809024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776924 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776899.424863:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776899.424864:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776899.424865:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776899.424868:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776899.424869:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776899.424870:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776899.424872:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776899.424873:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:0.0:1775776899.424875:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776899.424876:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776899.424877:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec380. 00000100:00000001:0.0:1775776899.424878:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776899.424880:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424882:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776899.424886:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776899.424887:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776899.424889:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776899.424890:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776900.123923:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776900.123930:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776900.123932:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776900.123938:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776900.123941:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776900.123945:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776900.123947:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776900.123950:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776900.123952:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776900.187947:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776900.187954:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776900.187960:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776900.187961:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776900.187963:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776900.187964:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776900.187967:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776900.187968:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776900.188325:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776900.188329:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776900.188332:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776900.315914:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776900.315922:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776900.315923:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776900.315927:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776900.315930:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776900.315940:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776900.315943:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776900.315945:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776900.315947:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776900.699953:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313288439, at 4313288448 00000100:00000010:1.0:1775776900.699970:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f2372a0. 00000100:00000040:0.0:1775776900.700055:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313288439, at 4313288448 00000100:00000010:0.0:1775776900.700061:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775776901.151323:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776901.151329:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776901.151332:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776901.151336:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776901.151340:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776901.151344:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776901.151347:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776901.151350:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776901.151352:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776901.211944:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211949:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776901.211953:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776901.211955:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776901.211958:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211961:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776901.211963:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211965:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211967:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211971:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776901.211973:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776901.211974:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776901.211975:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211977:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776901.211978:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211980:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211982:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211985:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776901.211986:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776901.211987:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776901.211988:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211990:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776901.211991:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211992:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776901.211994:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776901.212126:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776901.212129:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776901.212131:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776901.339919:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776901.339925:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776901.339927:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776901.339932:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776901.339935:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776901.339939:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776901.339942:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776901.339944:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776901.339946:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776902.172630:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776902.172636:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776902.172637:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776902.172642:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776902.172645:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776902.172648:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776902.172651:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776902.172653:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776902.172656:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776902.237230:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776902.237236:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776902.237240:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776902.237241:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776902.237243:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776902.237244:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776902.237246:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776902.237247:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776902.237339:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776902.237342:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776902.237345:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776902.363914:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776902.363922:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776902.363924:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776902.363929:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776902.363933:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776902.363937:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776902.363940:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776902.363942:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776902.363946:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776903.195912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776903.195920:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776903.195922:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776903.195927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776903.195931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776903.195934:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776903.195937:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776903.195939:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776903.195942:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776903.260928:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776903.260935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776903.260940:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776903.261345:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776903.261351:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776903.261353:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776903.261356:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776903.261357:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776903.261463:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776903.261466:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776903.261469:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776903.390108:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776903.390115:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776903.390117:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776903.390122:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776903.390126:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776903.390130:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776903.390132:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776903.390134:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776903.390137:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776904.219911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776904.219918:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776904.219920:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776904.219924:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776904.219927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776904.219930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776904.219932:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776904.219934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776904.219936:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776904.284211:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776904.284218:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776904.284224:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776904.284225:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776904.284229:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776904.284230:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776904.284234:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776904.284235:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776904.284292:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776904.284296:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776904.284299:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776904.411915:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776904.411922:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776904.411924:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776904.411930:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776904.411934:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776904.411938:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776904.411941:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776904.411943:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776904.411947:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776904.539945:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776904.539957:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776904.539960:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776904.539964:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776904.539968:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776904.539976:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18619 00000400:00000200:1.0:1775776904.539987:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776904.539993:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776904.539996:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776904.539999:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540003:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540005:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540008:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776904.540013:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776904.540014:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776904.540017:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776904.540020:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776904.540022:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776904.540024:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776904.540027:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540029:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540030:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540032:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776904.540035:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776904.540038:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540040:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776904.540043:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776904.540046:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776904.540049:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540052:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776904.540054:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776904.540064:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776904.540068:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776904.540071:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776904.540073:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540076:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776904.540078:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776904.540083:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776904.540092:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775776904.540098:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776904.540107:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776904.540117:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776904.540121:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776904.540126:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776904.540147:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776904.540150:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776904.540153:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776904.540154:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776904.540156:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776904.540158:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776904.540160:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18619 00000400:00000200:1.0:1775776904.540164:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776904.540167:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776904.540169:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776904.540172:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540174:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540176:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540178:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776904.540180:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776904.540182:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776904.540189:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776904.540192:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776904.540194:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776904.540196:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776904.540199:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540201:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540203:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540205:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776904.540207:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776904.540210:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540212:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776904.540214:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776904.540219:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776904.540221:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540223:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776904.540227:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776904.540230:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540232:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776904.540236:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776904.540238:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776904.540241:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776904.540242:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540244:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776904.540245:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776904.540247:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776904.540251:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775776904.540254:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776904.540259:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776904.540267:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776904.540269:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776904.540273:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776904.540281:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776904.540283:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776904.540287:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776904.540288:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776904.540290:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776904.540292:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776904.540295:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18619 00000400:00000200:1.0:1775776904.540298:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776904.540301:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776904.540303:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776904.540306:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540308:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540309:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540311:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776904.540313:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776904.540315:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776904.540316:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776904.540318:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776904.540320:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776904.540322:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776904.540325:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540327:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540328:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540330:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776904.540333:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776904.540335:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540338:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776904.540339:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776904.540342:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776904.540344:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540346:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776904.540348:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776904.540350:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540352:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776904.540355:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776904.540358:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776904.540360:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776904.540362:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540363:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776904.540364:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776904.540366:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776904.540369:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38c00. 00000100:00000001:1.0:1775776904.540372:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776904.540376:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776904.540383:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776904.540385:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776904.540389:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776904.540397:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776904.540399:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776904.540402:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776904.540404:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776904.540405:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776904.540408:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776904.540410:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18619 00000400:00000200:1.0:1775776904.540414:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776904.540416:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776904.540418:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776904.540420:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540422:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540424:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540426:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776904.540428:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776904.540429:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776904.540431:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776904.540433:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776904.540435:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776904.540437:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776904.540440:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540442:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540444:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540446:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776904.540448:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776904.540451:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540454:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776904.540455:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776904.540458:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776904.540460:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540462:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776904.540464:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776904.540466:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540468:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776904.540471:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776904.540473:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776904.540475:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776904.540477:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540478:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776904.540479:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776904.540481:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776904.540484:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775776904.540487:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776904.540491:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776904.540498:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776904.540500:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776904.540503:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776904.540511:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776904.540513:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776904.540517:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776904.540518:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776904.540520:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776904.540522:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776904.540525:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18619 00000400:00000200:1.0:1775776904.540529:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776904.540532:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776904.540534:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776904.540536:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540539:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540540:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540542:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776904.540545:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776904.540546:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776904.540547:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776904.540549:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776904.540551:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776904.540553:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776904.540556:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540558:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776904.540560:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540561:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776904.540563:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776904.540566:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776904.540568:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776904.540570:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776904.540573:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776904.540575:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540577:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776904.540578:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776904.540581:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540582:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776904.540586:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776904.540588:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776904.540590:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776904.540592:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776904.540593:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776904.540594:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776904.540596:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776904.540599:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775776904.540602:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776904.540606:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776904.540613:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776904.540615:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776904.540619:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776904.540627:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776904.540631:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18629) 00000100:00000001:0.0:1775776904.541942:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776904.541948:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776904.541951:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776904.541961:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.541972:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776904.541974:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.541978:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776904.541981:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776904.541983:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776904.541987:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099809152:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776904.541991:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776904.541995:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776904.541998:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.542000:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776904.542007:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776904.542009:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776904.542016:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776904.542020:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776904.542026:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776904.542034:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099809152, portal 25 00000100:00000001:0.0:1775776904.542037:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776904.542038:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287599232 : -114841421952384 : ffff978d65d6ca80) 00000100:00000040:0.0:1775776904.542044:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542052:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776904.542054:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776904.542057:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776904.542060:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099809152, offset 0 00000400:00000200:0.0:1775776904.542067:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776904.542074:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.542081:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776904.542086:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776904.542091:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776904.542094:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776904.542097:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776904.542101:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4552:4552:256:4294967295] 192.168.201.139@tcp LPNI seq info [4552:4552:8:4294967295] 00000400:00000200:0.0:1775776904.542105:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.542114:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776904.542120:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.542125:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776904.542131:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776904.542134:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776904.542139:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776904.542142:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776904.542159:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.542162:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776904.542168:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776904.542172:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776904.542176:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776904.542178:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776904.542182:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776904.542186:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542201:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.542204:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542211:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.542213:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776904.542218:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776904.542222:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776904.542225:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313293559 00000100:00000010:0.0:1775776904.542229:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776904.542237:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776904.542244:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313293559 00000100:00000010:0.0:1775776904.542247:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ae0. 00000100:00000001:0.0:1775776904.542250:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776904.542254:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776904.542256:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.542259:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776904.542263:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776904/real 1775776904] req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776904.542271:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776904.542273:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776904.542275:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776904.542277:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776904.542279:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776904.542281:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776904.542284:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776904.542292:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542298:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776904.542300:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542302:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.542303:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776904.542306:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542312:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776904.542316:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542323:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776904.542325:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542327:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.542327:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542330:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776904.542332:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776904.542335:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776904.542339:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776904.542341:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776904.542345:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776904.542352:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099809152:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776904.542357:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776904.542360:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542366:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776904.542367:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.542370:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776904.542371:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776904.542373:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776904.542375:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809280:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776904.542378:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776904.542380:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776904.542382:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.542383:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776904.542386:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776904.542388:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776904.542392:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776904.542394:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776904.542397:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776904.542400:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099809280, portal 10 00000100:00000001:0.0:1775776904.542402:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776904.542403:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287604608 : -114841421947008 : ffff978d65d6df80) 00000100:00000040:0.0:1775776904.542409:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542416:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776904.542418:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776904.542420:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776904.542422:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099809280, offset 0 00000400:00000200:0.0:1775776904.542427:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776904.542430:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.542433:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776904.542436:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776904.542439:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776904.542441:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776904.542443:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776904.542446:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4553:4553:256:4294967295] 192.168.201.139@tcp LPNI seq info [4553:4553:8:4294967295] 00000400:00000200:0.0:1775776904.542450:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.542456:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776904.542461:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.542464:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776904.542468:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776904.542471:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776904.542475:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776904.542477:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776904.542481:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.542483:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776904.542485:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776904.542489:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776904.542491:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776904.542492:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776904.542494:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776904.542497:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542503:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.542506:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542512:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.542514:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776904.542516:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776904.542520:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776904.542523:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776904.542525:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313292294 00000100:00000001:0.0:1775776904.542528:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313293559 00000100:00000001:0.0:1775776904.542530:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313292294 00000100:00000001:0.0:1775776904.542532:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313293559 00000100:00000001:0.0:1775776904.542534:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776904.542536:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.542538:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776904.542542:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776904/real 1775776904] req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776904.542548:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776904.542549:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776904.542551:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776904.542553:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776904.542555:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776904.542556:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776904.542559:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776904.542567:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542573:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776904.542575:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542577:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.542578:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776904.542581:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542587:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776904.542591:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542596:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776904.542598:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542600:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.542601:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542603:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776904.542604:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776904.542607:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776904.542610:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776904.542612:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776904.542615:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776904.542623:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809280:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776904.542627:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776904.542630:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542636:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776904.542638:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.542640:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776904.542641:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776904.542643:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776904.542646:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809408:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776904.542649:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776904.542651:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776904.542653:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.542654:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776904.542658:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776904.542660:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776904.542662:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776904.542665:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776904.542682:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776904.542685:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099809408, portal 10 00000100:00000001:0.0:1775776904.542687:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776904.542688:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601920 : -114841421949696 : ffff978d65d6d500) 00000100:00000040:0.0:1775776904.542693:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542698:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776904.542700:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776904.542702:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776904.542704:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099809408, offset 0 00000400:00000200:0.0:1775776904.542708:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776904.542711:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.542714:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776904.542716:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776904.542719:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776904.542721:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776904.542723:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776904.542725:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4554:4554:256:4294967295] 192.168.201.139@tcp LPNI seq info [4554:4554:8:4294967295] 00000400:00000200:0.0:1775776904.542728:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.542734:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776904.542738:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.542741:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776904.542745:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776904.542747:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776904.542750:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776904.542752:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776904.542771:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.542773:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776904.542776:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776904.542814:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776904.542818:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776904.542820:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776904.542822:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776904.542826:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542832:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.542835:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542841:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.542843:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776904.542846:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776904.542848:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776904.542851:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776904.542853:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313292296 00000100:00000001:0.0:1775776904.542855:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313293559 00000100:00000001:0.0:1775776904.542857:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313292296 00000100:00000001:0.0:1775776904.542860:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313293559 00000100:00000001:0.0:1775776904.542861:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776904.542863:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.542865:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776904.542868:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776904/real 1775776904] req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776904.542875:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776904.542876:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776904.542877:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776904.542879:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776904.542881:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776904.542882:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776904.542885:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776904.542892:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542899:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776904.542900:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542902:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.542903:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776904.542907:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542915:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776904.542918:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542924:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776904.542926:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542928:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.542929:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.542931:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776904.542932:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776904.542935:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776904.542938:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776904.542940:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776904.542944:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776904.542952:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809408:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776904.542955:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776904.542959:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.542965:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776904.542966:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.542968:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776904.542970:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776904.542972:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776904.542974:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809536:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776904.542977:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776904.542979:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776904.542980:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.542982:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776904.542986:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775776904.542988:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776904.542991:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776904.542993:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776904.542997:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776904.542999:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099809536, portal 4 00000100:00000001:0.0:1775776904.543001:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776904.543002:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287606400 : -114841421945216 : ffff978d65d6e680) 00000100:00000040:0.0:1775776904.543007:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543013:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776904.543015:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776904.543017:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776904.543019:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099809536, offset 0 00000400:00000200:0.0:1775776904.543023:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776904.543026:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.543029:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776904.543032:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776904.543034:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776904.543036:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776904.543038:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776904.543041:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4555:4555:256:4294967295] 192.168.201.139@tcp LPNI seq info [4555:4555:8:4294967295] 00000400:00000200:0.0:1775776904.543044:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.543049:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776904.543053:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.543056:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776904.543059:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776904.543061:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776904.543065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776904.543068:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776904.543071:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.543073:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776904.543075:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776904.543078:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776904.543080:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776904.543082:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776904.543084:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776904.543087:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543093:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.543096:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543107:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.543109:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776904.543111:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776904.543114:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776904.543117:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776904.543119:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313292305 00000100:00000001:0.0:1775776904.543121:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313293559 00000100:00000001:0.0:1775776904.543122:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313292305 00000100:00000001:0.0:1775776904.543124:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313293559 00000100:00000001:0.0:1775776904.543126:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776904.543128:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.543130:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776904.543133:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776904/real 1775776904] req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776904.543139:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776904.543139:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776904.543141:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776904.543142:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776904.543143:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776904.543145:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776904.543147:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776904.543151:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543155:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776904.543155:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543156:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.543157:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776904.543159:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543163:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776904.543165:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543169:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776904.543170:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543171:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.543171:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543172:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776904.543174:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776904.543175:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776904.543177:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776904.543178:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776904.543179:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776904.543184:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809536:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776904.543187:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776904.543190:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543197:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776904.543198:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.543200:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776904.543202:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776904.543204:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776904.543206:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809664:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776904.543209:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776904.543211:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776904.543213:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776904.543214:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776904.543217:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776904.543219:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776904.543222:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776904.543224:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776904.543228:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776904.543230:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099809664, portal 4 00000100:00000001:0.0:1775776904.543232:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776904.543233:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287607296 : -114841421944320 : ffff978d65d6ea00) 00000100:00000040:0.0:1775776904.543238:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543244:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776904.543246:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776904.543248:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776904.543251:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099809664, offset 0 00000400:00000200:0.0:1775776904.543255:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776904.543258:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.543261:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776904.543263:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776904.543264:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776904.543266:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776904.543267:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776904.543269:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4556:4556:256:4294967295] 192.168.201.139@tcp LPNI seq info [4556:4556:8:4294967295] 00000400:00000200:0.0:1775776904.543271:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776904.543275:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776904.543278:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.543280:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776904.543282:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776904.543284:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776904.543287:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776904.543288:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776904.543290:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776904.543292:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:0.0:1775776904.543294:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776904.543296:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776904.543298:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:0.0:1775776904.543299:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:0.0:1775776904.543300:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776904.543302:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543307:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.543309:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543313:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.543314:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776904.543315:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776904.543317:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776904.543319:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776904.543320:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313292296 00000100:00000001:0.0:1775776904.543321:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313293559 00000100:00000001:0.0:1775776904.543322:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313292296 00000100:00000001:0.0:1775776904.543324:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313293559 00000100:00000001:0.0:1775776904.543325:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776904.543326:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.543327:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776904.543329:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776904/real 1775776904] req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776904.543333:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776904.543334:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776904.543335:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776904.543336:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776904.543337:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776904.543339:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776904.543340:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776904.543345:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543348:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776904.543349:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543350:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.543350:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776904.543352:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543355:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776904.543357:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543361:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776904.543362:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543363:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776904.543363:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543364:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776904.543365:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776904.543367:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776904.543369:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776904.543370:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776904.543372:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776904.543376:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809664:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776904.543378:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543379:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.543381:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ca80 x1862033099809152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543386:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776904.543387:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776904.543389:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776904.543391:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776904.543392:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776904.543393:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776904.543395:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776904.543396:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39800. 02000000:00000001:0.0:1775776904.543399:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776904.543400:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776904.543401:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ca80. 00000100:00000001:0.0:1775776904.543404:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776904.543404:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543405:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.543407:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6df80 x1862033099809280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543411:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776904.543411:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776904.543412:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776904.543413:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776904.543414:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776904.543415:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776904.543415:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776904.543416:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38400. 02000000:00000001:0.0:1775776904.543417:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776904.543418:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776904.543418:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6df80. 00000100:00000001:0.0:1775776904.543420:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776904.543420:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543421:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.543423:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d500 x1862033099809408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543426:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776904.543427:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776904.543428:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776904.543429:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776904.543430:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776904.543431:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776904.543432:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776904.543432:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38c00. 02000000:00000001:0.0:1775776904.543433:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776904.543434:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776904.543434:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d500. 00000100:00000001:0.0:1775776904.543435:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776904.543436:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543437:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.543438:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6e680 x1862033099809536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543441:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776904.543442:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776904.543443:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775776904.543444:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776904.543445:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776904.543446:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776904.543446:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776904.543447:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39400. 02000000:00000001:0.0:1775776904.543448:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776904.543449:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776904.543449:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6e680. 00000100:00000001:0.0:1775776904.543450:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776904.543451:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543452:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776904.543453:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ea00 x1862033099809664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776929 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776904.543456:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776904.543457:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776904.543457:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776904.543458:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776904.543459:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776904.543460:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776904.543461:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776904.543462:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a400. 02000000:00000001:0.0:1775776904.543463:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776904.543464:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776904.543464:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ea00. 00000100:00000001:0.0:1775776904.543465:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776904.543466:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543467:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776904.543473:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776904.543474:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776904.543475:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776904.543476:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776905.244171:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776905.244177:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776905.244179:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776905.244184:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776905.244186:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776905.244189:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776905.244191:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776905.244193:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776905.244195:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776905.307924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776905.307930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776905.307936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776905.307937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776905.307941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776905.307942:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776905.307945:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776905.307946:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776905.308113:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776905.308116:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776905.308119:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776905.435902:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776905.435910:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776905.435911:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776905.435916:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776905.435920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776905.435924:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776905.435927:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776905.435929:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776905.435932:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776905.819894:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313293559, at 4313293568 00000100:00000010:0.0:1775776905.819908:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775776905.820019:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313293559, at 4313293568 00000100:00000010:1.0:1775776905.820024:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ae0. 00000100:00000001:3.0:1775776906.267903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776906.267910:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776906.267912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776906.267917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776906.267921:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776906.267924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776906.267927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776906.267929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776906.267932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776906.331909:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776906.331923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776906.331932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776906.331933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776906.331935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776906.331936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776906.331938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776906.331939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776906.331994:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776906.331998:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776906.332002:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776906.459893:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776906.459899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776906.459901:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776906.459906:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776906.459909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776906.459913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776906.459915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776906.459917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776906.459920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776907.292264:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776907.292271:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776907.292273:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776907.292278:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776907.292281:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776907.292284:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776907.292287:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776907.292289:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776907.292291:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776907.356100:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776907.356106:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776907.356111:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776907.356112:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776907.356116:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776907.356117:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776907.356120:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776907.356122:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776907.356587:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776907.356591:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776907.356594:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776907.483900:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776907.483907:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776907.483909:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776907.483915:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776907.483919:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776907.483922:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776907.483925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776907.483928:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776907.483931:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776908.315920:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776908.315926:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776908.315928:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776908.315932:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776908.315935:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776908.315939:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776908.315941:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776908.315943:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776908.315945:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776908.379943:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776908.379951:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776908.379958:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776908.379959:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776908.379963:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776908.379964:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776908.379968:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776908.379969:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776908.380593:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776908.380596:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776908.380600:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776908.507891:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776908.507897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776908.507899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776908.507904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776908.507907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776908.507911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776908.507914:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776908.507916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776908.507919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.340034:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776909.340042:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776909.340043:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776909.340049:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776909.340053:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776909.340057:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776909.340059:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776909.340061:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776909.340064:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776909.404968:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776909.404977:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776909.404980:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776909.404983:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776909.404988:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776909.404990:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776909.404993:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776909.404995:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776909.404999:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776909.405000:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776909.405004:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776909.405005:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776909.405009:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776909.405010:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776909.405417:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776909.405421:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776909.405423:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776909.531892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776909.531898:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776909.531900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776909.531905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776909.531909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776909.531913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776909.531915:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776909.531917:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776909.531920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776909.659965:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776909.659975:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776909.659978:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776909.659983:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776909.659987:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776909.659994:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18624 00000400:00000200:3.0:1775776909.660004:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776909.660011:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776909.660014:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776909.660017:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.660020:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.660022:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.660025:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776909.660030:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776909.660032:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776909.660034:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776909.660037:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776909.660039:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776909.660041:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776909.660044:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.660047:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.660049:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.660051:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776909.660053:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776909.660056:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776909.660058:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776909.660061:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776909.660065:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776909.660068:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.660070:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776909.660072:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776909.660081:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776909.660084:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776909.660087:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776909.660089:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.660092:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776909.660094:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776909.660098:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776909.660105:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776909.660112:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776909.660120:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776909.660131:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776909.660134:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776909.660139:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.660484:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776909.660488:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776909.660490:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776909.660496:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.660504:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776909.660505:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.660509:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776909.660511:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776909.660513:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776909.660516:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099809792:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776909.660520:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776909.660524:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776909.660527:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.660529:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776909.660534:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776909.660536:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776909.660541:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776909.660544:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776909.660562:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776909.660569:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099809792, portal 25 00000100:00000001:0.0:1775776909.660571:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776909.660572:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751260672 : -114841958290944 : ffff978d45deea00) 00000100:00000040:0.0:1775776909.660578:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.660585:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776909.660587:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776909.660590:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8ee0. 00000100:00000200:0.0:1775776909.660592:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099809792, offset 0 00000400:00000200:0.0:1775776909.660598:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776909.660605:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.660611:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776909.660615:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776909.660619:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776909.660623:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776909.660625:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776909.660629:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4557:4557:256:4294967295] 192.168.201.139@tcp LPNI seq info [4557:4557:8:4294967295] 00000400:00000200:0.0:1775776909.660633:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.660642:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776909.660648:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776909.660653:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bb00. 00000800:00000200:0.0:1775776909.660659:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776909.660664:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776909.660714:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776909.660719:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776909.660722:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313298677 00000100:00000010:0.0:1775776909.660727:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776909.660733:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776909.660740:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313298677 00000100:00000010:0.0:1775776909.660744:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ae0. 00000100:00000001:0.0:1775776909.660747:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776909.660750:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776909.660752:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.660755:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.660758:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776909.661172:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776909.661177:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776909.661181:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776909.661183:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776909.661185:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776909.661187:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776909.661191:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18624 00000400:00000200:3.0:1775776909.661196:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776909.661200:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776909.661202:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776909.661205:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.661207:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.661210:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.661212:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776909.661215:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776909.661216:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776909.661218:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776909.661220:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776909.661222:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776909.661224:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776909.661228:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.661230:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.661232:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.661234:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776909.661236:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776909.661239:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776909.661241:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776909.661243:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776909.661248:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776909.661250:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.661252:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776909.661256:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776909.661258:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.661260:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776909.661264:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776909.661267:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776909.661269:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776909.661271:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.661274:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776909.661276:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776909.661278:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776909.661283:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776909.661286:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776909.661292:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776909.661317:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776909.661320:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776909.661324:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.663168:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776909.663183:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776909.663188:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776909.663190:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776909.663193:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776909.663203:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.663213:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776909.663214:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.663217:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776909.663220:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776909.663222:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776909.663225:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809920:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776909.663228:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776909.663232:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776909.663234:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.663236:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776909.663244:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776909.663246:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776909.663252:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776909.663259:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513067e8. 00000400:00000010:0.0:1775776909.663264:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776909.663269:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099809920, portal 10 00000100:00000001:0.0:1775776909.663271:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776909.663273:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751254400 : -114841958297216 : ffff978d45ded180) 00000100:00000040:0.0:1775776909.663279:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.663285:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776909.663288:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776909.663290:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8f68. 00000100:00000200:0.0:1775776909.663302:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099809920, offset 0 00000400:00000200:0.0:1775776909.663317:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776909.663324:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.663330:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776909.663334:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776909.663338:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776909.663341:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776909.663344:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776909.663348:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4558:4558:255:4294967295] 192.168.201.139@tcp LPNI seq info [4558:4558:7:4294967295] 00000400:00000200:0.0:1775776909.663352:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.663360:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776909.663365:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776909.663370:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b400. 00000800:00000200:0.0:1775776909.663376:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776909.663380:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776909.663384:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776909.663388:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776909.663393:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776909.663395:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313297413 00000100:00000001:0.0:1775776909.663398:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313298677 00000100:00000001:0.0:1775776909.663400:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313297413 00000100:00000001:0.0:1775776909.663402:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313298677 00000100:00000001:0.0:1775776909.663404:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776909.663406:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.663409:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.663411:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776909.663415:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776909.663416:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776909.663418:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776909.663419:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776909.663421:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.663423:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.663513:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776909.663518:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776909.663523:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776909.663524:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776909.663527:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776909.663530:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776909.663533:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18624 00000400:00000200:3.0:1775776909.663537:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776909.663540:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776909.663543:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776909.663546:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.663549:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.663551:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.663553:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776909.663556:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776909.663557:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776909.663559:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776909.663561:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776909.663563:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776909.663566:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776909.663568:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.663570:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.663571:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.663574:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776909.663576:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776909.663578:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776909.663580:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776909.663582:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776909.663586:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776909.663588:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.663590:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776909.663593:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776909.663596:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.663597:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776909.663602:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776909.663605:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776909.663608:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776909.663609:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.663612:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776909.663614:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776909.663617:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776909.663623:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776909.663627:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776909.663633:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776909.663640:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776909.663644:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776909.663648:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776909.663656:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776909.663659:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776909.663662:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776909.663664:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776909.663675:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776909.663678:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776909.663681:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18624 00000400:00000200:3.0:1775776909.663684:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776909.663687:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776909.663689:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776909.665441:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665450:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.665456:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665458:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776909.665461:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776909.665463:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776909.665465:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776909.665467:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776909.665470:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776909.665472:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776909.665475:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665476:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.665478:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665479:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776909.665482:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776909.665485:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665488:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776909.665490:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776909.665495:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776909.665497:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.665499:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776909.665503:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776909.665505:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.665507:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776909.665514:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776909.665517:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776909.665520:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776909.665521:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.665524:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776909.665526:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776909.665529:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776909.665534:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776909.665539:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776909.665556:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776909.665565:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776909.665568:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776909.665572:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776909.665580:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776909.665583:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776909.665587:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776909.665589:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776909.665591:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776909.665593:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776909.665596:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18624 00000400:00000200:3.0:1775776909.665602:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776909.665607:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776909.665610:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776909.665612:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665614:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.665615:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665618:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776909.665619:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776909.665621:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776909.665622:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776909.665624:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776909.665625:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776909.665627:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776909.665629:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665631:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776909.665632:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665634:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776909.665636:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776909.665638:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776909.665640:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776909.665642:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776909.665644:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776909.665646:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.665647:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776909.665649:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776909.665651:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.665652:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776909.665656:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776909.665658:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776909.665660:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776909.665661:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776909.665662:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776909.665664:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776909.665677:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776909.665688:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776909.665698:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776909.665703:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776909.665715:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776909.665716:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776909.665719:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776909.665726:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776909.665730:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18635) 00000100:00000001:0.0:1775776909.665953:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776909.665957:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776909.665959:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776909.665964:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.665972:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776909.665973:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.665976:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776909.665978:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776909.665980:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776909.665983:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810048:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776909.665986:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776909.665989:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776909.665991:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.665993:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776909.665998:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776909.666000:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776909.666005:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776909.666009:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306e70. 00000400:00000010:0.0:1775776909.666015:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8880. 00000100:00000200:0.0:1775776909.666020:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099810048, portal 10 00000100:00000001:0.0:1775776909.666023:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776909.666024:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751249920 : -114841958301696 : ffff978d45dec000) 00000100:00000040:0.0:1775776909.666030:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.666037:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776909.666040:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776909.666042:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8a18. 00000100:00000200:0.0:1775776909.666045:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099810048, offset 0 00000400:00000200:0.0:1775776909.666051:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776909.666058:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.666063:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776909.666067:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776909.666070:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776909.666073:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776909.666075:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776909.666079:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4559:4559:254:4294967295] 192.168.201.139@tcp LPNI seq info [4559:4559:6:4294967295] 00000400:00000200:0.0:1775776909.666083:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.666090:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776909.666095:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776909.666099:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775776909.666105:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776909.666109:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776909.666113:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776909.666117:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776909.666119:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776909.666121:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313297419 00000100:00000001:0.0:1775776909.666124:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313298677 00000100:00000001:0.0:1775776909.666125:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313297419 00000100:00000001:0.0:1775776909.666127:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313298677 00000100:00000001:0.0:1775776909.666129:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776909.666131:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.666133:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776909.666136:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.666142:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776909.666143:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.666145:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776909.666147:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776909.666149:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776909.666151:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810176:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776909.666154:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776909.666155:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776909.666157:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.666158:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776909.666161:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775776909.666162:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776909.666165:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776909.666167:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306c60. 00000400:00000010:0.0:1775776909.666170:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8b28. 00000100:00000200:0.0:1775776909.666172:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099810176, portal 4 00000100:00000001:0.0:1775776909.666174:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776909.666175:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751255296 : -114841958296320 : ffff978d45ded500) 00000100:00000040:0.0:1775776909.666179:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.666184:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776909.666185:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776909.666187:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8dd0. 00000100:00000200:0.0:1775776909.666189:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099810176, offset 0 00000400:00000200:0.0:1775776909.666192:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776909.666195:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.666198:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776909.666200:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776909.666203:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776909.666205:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776909.666207:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776909.666209:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4560:4560:253:4294967295] 192.168.201.139@tcp LPNI seq info [4560:4560:5:4294967295] 00000400:00000200:0.0:1775776909.666212:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.666216:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776909.666220:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776909.666224:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775776909.666228:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776909.666231:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776909.666233:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776909.666235:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776909.666237:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776909.666239:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313297428 00000100:00000001:0.0:1775776909.666240:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313298677 00000100:00000001:0.0:1775776909.666242:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313297428 00000100:00000001:0.0:1775776909.666244:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313298677 00000100:00000001:0.0:1775776909.666245:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776909.666247:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.666248:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776909.666251:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.666257:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776909.666258:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.666259:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776909.666261:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776909.666262:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776909.666264:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810304:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776909.666267:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776909.666269:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776909.666270:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776909.666271:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776909.666274:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776909.666275:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776909.666277:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776909.666279:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069f8. 00000400:00000010:0.0:1775776909.666282:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b85d8. 00000100:00000200:0.0:1775776909.666284:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099810304, portal 4 00000100:00000001:0.0:1775776909.666285:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776909.666286:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751258880 : -114841958292736 : ffff978d45dee300) 00000100:00000040:0.0:1775776909.666290:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776909.666295:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776909.666297:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776909.666299:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8088. 00000100:00000200:0.0:1775776909.666301:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099810304, offset 0 00000400:00000200:0.0:1775776909.666305:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776909.666308:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.666310:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776909.666313:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776909.666315:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776909.666317:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776909.666318:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776909.666321:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4561:4561:252:4294967295] 192.168.201.139@tcp LPNI seq info [4561:4561:4:4294967295] 00000400:00000200:0.0:1775776909.666324:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776909.666329:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776909.666333:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776909.666335:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b300. 00000800:00000200:0.0:1775776909.666339:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776909.666341:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776909.666343:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776909.666346:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776909.666348:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776909.666349:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313297419 00000100:00000001:0.0:1775776909.666352:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313298677 00000100:00000001:0.0:1775776909.666353:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313297419 00000100:00000001:0.0:1775776909.666355:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313298677 00000100:00000001:0.0:1775776909.666356:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776909.666358:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.666361:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.666363:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776909.666368:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776909.666370:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776909.666372:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776909.666373:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776909.666374:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776909.666376:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776910.363946:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776910.363954:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776910.363956:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776910.363961:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776910.363965:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776910.363969:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776910.363971:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776910.363974:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776910.363976:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776910.428048:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776910.428054:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776910.428060:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776910.428061:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776910.428065:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776910.428066:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776910.428069:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776910.428070:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776910.428170:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776910.428173:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776910.428176:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776910.555968:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776910.555975:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776910.555977:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776910.555983:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776910.555987:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776910.555991:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776910.555993:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776910.555996:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776910.555999:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776910.939916:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313298677, at 4313298688 00000100:00000040:1.0:1775776910.939935:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313298677, at 4313298688 00000100:00000010:0.0:1775776910.939945:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000010:1.0:1775776910.939947:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ae0. 00000100:00000001:3.0:1775776911.388371:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776911.388379:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776911.388381:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776911.388387:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776911.388390:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776911.388394:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776911.388397:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776911.388399:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776911.388402:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776911.451933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451940:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776911.451943:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776911.451945:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776911.451949:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451952:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776911.451954:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451956:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451958:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451962:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776911.451964:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776911.451965:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776911.451966:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451968:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776911.451970:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451971:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451973:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451977:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776911.451978:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776911.451980:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776911.451981:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451982:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776911.451984:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451985:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776911.451987:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776911.452375:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776911.452379:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776911.452382:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776911.579919:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776911.579926:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776911.579929:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776911.579934:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776911.579937:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776911.579941:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776911.579944:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776911.579946:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776911.579949:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776912.411881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776912.411889:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776912.411891:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776912.411898:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776912.411901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776912.411906:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776912.411908:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776912.411911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776912.411914:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776912.477059:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776912.477066:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776912.477072:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776912.477073:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776912.477076:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776912.477078:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776912.477081:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776912.477082:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776912.477222:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776912.477226:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776912.477229:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776912.605108:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776912.605115:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776912.605117:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776912.605122:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776912.605126:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776912.605130:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776912.605132:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776912.605135:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776912.605138:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775776912.732335:0:301879:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:2.0:1775776912.732385:0:301879:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775776912.732394:0:301879:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776912.732397:0:301879:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:2.0:1775776912.732401:0:301879:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776912.732405:0:301879:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775776912.732408:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:2.0:1775776912.732411:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bb00. 00000400:00000200:2.0:1775776912.732416:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776912.732422:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776912.732429:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8ee0 00000400:00000010:2.0:1775776912.732431:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8ee0. 00000100:00000001:2.0:1775776912.732436:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776912.732445:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732479:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776912.732484:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732491:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776912.732493:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776912.732501:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:2.0:1775776912.732504:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b400. 00000400:00000200:2.0:1775776912.732507:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776912.732510:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776912.732512:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8f68 00000400:00000010:2.0:1775776912.732514:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8f68. 00000100:00000001:2.0:1775776912.732515:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776912.732518:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732524:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776912.732526:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732530:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776912.732531:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776912.732534:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:2.0:1775776912.732536:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:2.0:1775776912.732537:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776912.732539:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776912.732540:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8a18 00000400:00000010:2.0:1775776912.732541:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8a18. 00000100:00000001:2.0:1775776912.732542:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776912.732546:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732552:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776912.732554:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732560:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776912.732561:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776912.732564:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:2.0:1775776912.732566:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:2.0:1775776912.732569:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776912.732572:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776912.732575:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8dd0 00000400:00000010:2.0:1775776912.732576:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8dd0. 00000100:00000001:2.0:1775776912.732578:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776912.732582:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732586:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776912.732588:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732592:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776912.732592:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776912.732595:0:301879:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:2.0:1775776912.732596:0:301879:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b300. 00000400:00000200:2.0:1775776912.732598:0:301879:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776912.732601:0:301879:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776912.732603:0:301879:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8088 00000400:00000010:2.0:1775776912.732605:0:301879:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8088. 00000100:00000001:2.0:1775776912.732607:0:301879:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776912.732610:0:301879:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732628:0:301879:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776912.732632:0:301879:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776934 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776912.732637:0:301879:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776912.732639:0:301879:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776912.732674:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776912.732679:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776912.732682:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776912.732689:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776909/real 1775776912] req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776912.732702:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776912.732703:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776912.732706:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306e70. 00000400:00000200:0.0:1775776912.732710:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8880 00000400:00000010:0.0:1775776912.732712:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8880. 00000100:00000001:0.0:1775776912.732715:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776912.732718:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776912.732728:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732733:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776912.732735:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732738:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.732740:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776912.732744:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732755:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776912.732760:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732767:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776912.732770:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732771:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.732773:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732776:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776912.732799:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776912.732804:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776912.732809:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776912.732811:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776912.732815:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776912.732822:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810048:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776912.732827:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776912.732830:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776909/real 1775776912] req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776912.732838:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776912.732839:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776912.732841:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306c60. 00000400:00000200:0.0:1775776912.732843:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8b28 00000400:00000010:0.0:1775776912.732845:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8b28. 00000100:00000001:0.0:1775776912.732846:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776912.732849:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776912.732860:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732866:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776912.732867:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732869:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.732870:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776912.732873:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732879:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776912.732882:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732888:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776912.732890:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732892:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.732893:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732894:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776912.732896:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776912.732899:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776912.732902:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776912.732904:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776912.732907:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776912.732913:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810176:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776912.732916:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776912.732920:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776909/real 1775776912] req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776912.732932:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776912.732933:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776912.732935:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069f8. 00000400:00000200:0.0:1775776912.732937:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b85d8 00000400:00000010:0.0:1775776912.732938:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b85d8. 00000100:00000001:0.0:1775776912.732940:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776912.732943:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776912.732951:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732956:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776912.732958:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732959:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.732960:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776912.732963:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732969:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776912.732971:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.732977:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776912.732978:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732980:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.732981:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.732982:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776912.732984:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776912.732986:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776912.732989:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776912.732991:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776912.732995:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776912.733001:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810304:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776912.733006:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776912.733009:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776909/real 1775776912] req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776912.733017:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776912.733018:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776912.733019:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513067e8. 00000400:00000200:0.0:1775776912.733022:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776912.733024:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776912.733026:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776912.733029:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776912.733036:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733041:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776912.733042:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733044:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.733046:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776912.733049:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733055:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776912.733059:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733064:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776912.733066:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733069:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.733070:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733071:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776912.733073:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776912.733075:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776912.733078:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776912.733080:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776912.733083:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776912.733091:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099809920:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776912.733095:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776912.733098:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776909/real 1775776912] req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776912.733105:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776912.733106:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776912.733108:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776912.733110:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776912.733111:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776912.733113:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776912.733116:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776912.733122:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733128:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776912.733129:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733130:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.733131:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776912.733134:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733146:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776912.733149:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733155:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776912.733157:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733158:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776912.733159:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733160:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776912.733162:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776912.733164:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776912.733167:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776912.733169:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776912.733172:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776912.733180:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099809792:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776912.733184:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733186:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776912.733189:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099810048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733196:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776912.733199:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776912.733203:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775776912.733207:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776912.733210:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776912.733212:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776912.733215:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776912.733217:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:0.0:1775776912.733222:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776912.733224:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776912.733226:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:0.0:1775776912.733229:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776912.733230:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733232:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776912.733235:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded500 x1862033099810176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733240:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776912.733241:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776912.733243:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775776912.733245:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776912.733246:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776912.733247:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776912.733249:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776912.733251:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:0.0:1775776912.733252:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776912.733254:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776912.733255:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded500. 00000100:00000001:0.0:1775776912.733257:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776912.733258:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733259:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776912.733262:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee300 x1862033099810304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733268:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776912.733270:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776912.733271:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776912.733274:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776912.733275:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776912.733276:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776912.733278:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776912.733279:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:0.0:1775776912.733281:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776912.733282:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776912.733283:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee300. 00000100:00000001:0.0:1775776912.733286:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776912.733287:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733289:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776912.733292:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099809920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733297:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776912.733299:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776912.733300:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776912.733302:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776912.733303:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776912.733304:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776912.733306:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776912.733307:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:0.0:1775776912.733310:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776912.733311:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776912.733312:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:0.0:1775776912.733314:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776912.733315:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733316:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776912.733319:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099809792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776934 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776912.733325:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776912.733326:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776912.733327:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776912.733329:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776912.733330:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776912.733331:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776912.733333:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776912.733334:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:0.0:1775776912.733336:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776912.733336:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776912.733337:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:0.0:1775776912.733339:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776912.733340:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733342:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776912.733350:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776912.733351:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776912.733353:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776912.733355:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776913.435928:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776913.435935:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776913.435937:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776913.435943:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776913.435947:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776913.435950:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776913.435953:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776913.435955:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776913.435958:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776913.500002:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776913.500008:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776913.500014:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776913.500015:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776913.500018:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776913.500020:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776913.500023:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776913.500025:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776913.500192:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776913.500195:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776913.500198:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776913.627904:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776913.627911:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776913.627913:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776913.627919:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776913.627924:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776913.627928:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776913.627930:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776913.627933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776913.627935:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.459899:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776914.459907:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776914.459909:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776914.459915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776914.459918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776914.459922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776914.459924:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776914.459926:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776914.459929:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776914.524192:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776914.524198:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776914.524201:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776914.524202:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776914.524204:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776914.524205:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776914.524208:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776914.524208:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776914.524270:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776914.524274:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776914.524278:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776914.651882:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776914.651888:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776914.651889:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776914.651893:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776914.651896:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776914.651899:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776914.651901:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776914.651903:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776914.651904:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776914.779892:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776914.779904:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776914.779907:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776914.779911:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776914.779915:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776914.779922:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18630 00000400:00000200:3.0:1775776914.779932:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776914.779937:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776914.779940:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776914.779944:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.779947:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.779949:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.779953:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776914.779956:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776914.779958:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776914.779961:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776914.779963:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776914.779966:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776914.779968:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776914.779970:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.779972:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.779973:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.779975:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776914.779978:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776914.779981:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776914.779984:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776914.779987:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776914.779990:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776914.779993:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.779995:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776914.779997:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776914.780004:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775776914.780007:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776914.780011:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776914.780012:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.780015:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776914.780018:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776914.780022:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776914.780030:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776914.780036:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776914.780045:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776914.780056:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776914.780061:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776914.780066:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.780135:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776914.780139:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776914.780142:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776914.780148:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.780156:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776914.780158:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.780161:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776914.780163:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776914.780165:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776914.780168:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099810432:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776914.780172:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776914.780177:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776914.780179:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.780181:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776914.780187:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776914.780189:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776914.780194:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776914.780198:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776914.780204:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776914.780211:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099810432, portal 25 00000100:00000001:0.0:1775776914.780213:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776914.780215:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751264256 : -114841958287360 : ffff978d45def800) 00000100:00000040:0.0:1775776914.780220:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.780227:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776914.780229:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776914.780232:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776914.780235:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099810432, offset 0 00000400:00000200:0.0:1775776914.780242:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776914.780249:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.780255:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776914.780259:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776914.780264:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776914.780267:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776914.780270:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776914.780275:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4562:4562:256:4294967295] 192.168.201.139@tcp LPNI seq info [4562:4562:8:4294967295] 00000400:00000200:0.0:1775776914.780280:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.780290:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776914.780297:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.780303:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776914.780310:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776914.780314:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776914.780319:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776914.780322:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776914.780326:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.780329:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776914.780334:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776914.780338:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776914.780343:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776914.780346:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776914.780349:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776914.780353:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776914.780378:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776914.780381:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776914.780385:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776914.780386:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776914.780388:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776914.780390:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776914.780393:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18630 00000400:00000200:3.0:1775776914.780397:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776914.780400:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776914.780402:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776914.780405:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.780407:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.780408:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.780410:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776914.780423:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776914.780424:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776914.780426:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776914.780428:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776914.780431:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776914.780433:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776914.780436:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.780437:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.780439:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.780440:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776914.780443:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776914.780445:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776914.780447:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776914.780450:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776914.780455:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776914.780457:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.780459:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776914.780463:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776914.780465:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.780467:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776914.780471:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776914.780473:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776914.780475:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776914.780477:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.780479:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776914.780480:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776914.780482:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776914.780486:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776914.780489:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776914.780494:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776914.780502:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776914.780504:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776914.780507:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781356:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.781366:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781375:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.781379:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776914.781389:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776914.781395:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776914.781398:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313303798 00000100:00000010:0.0:1775776914.781404:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776914.781410:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776914.781418:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313303798 00000100:00000010:0.0:1775776914.781422:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f2372a0. 00000100:00000001:0.0:1775776914.781424:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776914.781426:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776914.781429:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.781431:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776914.781436:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776914/real 1775776914] req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776914.781445:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776914.781447:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776914.781449:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776914.781453:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776914.781455:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776914.781457:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776914.781461:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776914.781469:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781476:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776914.781477:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781479:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.781481:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776914.781485:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781491:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776914.781495:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781503:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776914.781506:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781508:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.781509:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781511:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776914.781513:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776914.781516:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776914.781521:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776914.781523:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776914.781527:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776914.781536:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099810432:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776914.781540:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781542:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.781545:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def800 x1862033099810432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781551:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776914.781554:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776914.781556:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776914.781560:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776914.781562:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776914.781563:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776914.781566:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776914.781569:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115ac00. 02000000:00000001:0.0:1775776914.781572:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776914.781574:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776914.781587:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def800. 00000100:00000001:0.0:1775776914.781591:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776914.781593:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781595:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781603:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776914.781605:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.781607:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776914.781609:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776914.781610:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776914.781615:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781627:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776914.781628:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.781631:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776914.781633:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776914.781635:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776914.781638:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810560:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776914.781642:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776914.781644:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776914.781646:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.781648:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776914.781652:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776914.781653:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776914.781658:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776914.781661:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776914.781665:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776914.781671:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099810560, portal 10 00000100:00000001:0.0:1775776914.781673:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776914.781674:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257984 : -114841958293632 : ffff978d45dedf80) 00000100:00000040:0.0:1775776914.781681:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781700:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776914.781703:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776914.781706:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776914.781709:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099810560, offset 0 00000400:00000200:0.0:1775776914.781713:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776914.781719:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.781723:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776914.781727:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776914.781730:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776914.781734:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776914.781736:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776914.781740:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4563:4563:256:4294967295] 192.168.201.139@tcp LPNI seq info [4563:4563:8:4294967295] 00000400:00000200:0.0:1775776914.781744:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.781751:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776914.781756:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.781760:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776914.781765:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776914.781768:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776914.781772:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776914.781776:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776914.781796:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.781799:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776914.781804:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776914.781807:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776914.781811:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776914.781812:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776914.781815:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776914.781818:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781826:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.781828:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781835:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.781836:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776914.781839:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776914.781843:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776914.781846:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776914.781848:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313302533 00000100:00000001:0.0:1775776914.781851:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313303798 00000100:00000001:0.0:1775776914.781853:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313302533 00000100:00000001:0.0:1775776914.781855:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313303798 00000100:00000001:0.0:1775776914.781858:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776914.781860:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.781862:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776914.781865:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776914/real 1775776914] req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776914.781873:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776914.781874:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776914.781876:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776914.781878:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776914.781879:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776914.781881:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776914.781884:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776914.781892:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781899:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776914.781900:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781902:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.781903:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776914.781906:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781912:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776914.781916:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781922:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776914.781925:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781926:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.781927:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781929:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776914.781932:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776914.781934:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776914.781938:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776914.781940:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776914.781944:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776914.781952:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810560:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776914.781956:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781957:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.781960:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedf80 x1862033099810560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.781966:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776914.781968:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776914.781969:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776914.781972:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776914.781973:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776914.781974:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776914.781976:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776914.781978:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a000. 02000000:00000001:0.0:1775776914.781980:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776914.781982:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776914.781983:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedf80. 00000100:00000001:0.0:1775776914.781986:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776914.781987:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781989:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.781992:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776914.781994:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.781996:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776914.781997:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786302:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776914.786312:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776914.786316:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776914.786318:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776914.786321:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776914.786325:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776914.786332:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18630 00000400:00000200:3.0:1775776914.786342:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776914.786348:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776914.786351:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776914.786354:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786356:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.786359:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786361:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776914.786365:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776914.786366:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776914.786369:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776914.786371:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776914.786374:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776914.786376:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776914.786379:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786380:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.786382:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786384:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776914.786387:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776914.786390:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786392:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776914.786395:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776914.786403:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776914.786405:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786407:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776914.786411:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776914.786413:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786415:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776914.786424:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775776914.786427:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776914.786430:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776914.786432:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786434:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776914.786437:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776914.786441:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776914.786450:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775776914.786456:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776914.786465:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776914.786474:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776914.786478:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776914.786482:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776914.786491:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776914.786493:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776914.786496:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776914.786497:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776914.786499:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776914.786501:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776914.786503:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18630 00000400:00000200:3.0:1775776914.786506:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776914.786509:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776914.786510:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776914.786513:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786515:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.786516:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786518:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776914.786520:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776914.786521:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776914.786522:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776914.786524:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776914.786525:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776914.786527:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776914.786529:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786531:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.786532:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786534:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776914.786536:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776914.786538:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786540:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776914.786541:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776914.786543:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776914.786545:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786546:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776914.786548:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776914.786550:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786551:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776914.786554:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775776914.786556:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776914.786558:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776914.786559:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786561:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776914.786562:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776914.786563:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776914.786566:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775776914.786568:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776914.786571:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776914.786590:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776914.786592:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776914.786595:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776914.786602:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776914.786604:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776914.786607:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776914.786608:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776914.786610:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776914.786612:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776914.786614:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18630 00000400:00000200:3.0:1775776914.786618:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776914.786621:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776914.786623:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776914.786625:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786627:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.786629:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786630:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776914.786632:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776914.786633:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776914.786635:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776914.786636:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776914.786638:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776914.786640:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776914.786643:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786644:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776914.786646:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786648:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776914.786650:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776914.786652:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776914.786654:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776914.786655:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776914.786658:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776914.786660:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786661:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776914.786663:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776914.786664:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786666:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776914.786669:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775776914.786671:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776914.786673:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776914.786674:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776914.786675:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776914.786677:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776914.786678:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776914.786681:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775776914.786684:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776914.786688:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776914.786695:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776914.786697:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776914.786700:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776914.786708:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776914.786713:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18640) 00000100:00000001:0.0:1775776914.786849:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776914.786854:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776914.786856:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776914.786862:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.786875:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776914.786878:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.786881:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776914.786884:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776914.786886:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776914.786889:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810688:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776914.786893:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776914.786897:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776914.786899:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.786902:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776914.786906:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776914.786908:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776914.786913:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776914.786917:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776914.786923:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776914.786929:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099810688, portal 10 00000100:00000001:0.0:1775776914.786931:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776914.786933:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257984 : -114841958293632 : ffff978d45dedf80) 00000100:00000040:0.0:1775776914.786939:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.786947:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776914.786950:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776914.786952:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776914.786955:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099810688, offset 0 00000400:00000200:0.0:1775776914.786961:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776914.786968:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.786973:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776914.786978:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776914.786982:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776914.786985:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776914.786988:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776914.786992:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4564:4564:256:4294967295] 192.168.201.139@tcp LPNI seq info [4564:4564:8:4294967295] 00000400:00000200:0.0:1775776914.786997:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.787004:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776914.787009:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.787014:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776914.787020:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776914.787024:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776914.787029:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776914.787032:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776914.787036:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.787038:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776914.787042:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776914.787046:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776914.787051:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776914.787054:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776914.787057:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776914.787061:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787070:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.787073:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787078:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.787079:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776914.787083:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776914.787087:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776914.787089:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776914.787091:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313302540 00000100:00000001:0.0:1775776914.787093:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313303798 00000100:00000001:0.0:1775776914.787094:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313302540 00000100:00000001:0.0:1775776914.787096:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313303798 00000100:00000001:0.0:1775776914.787097:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776914.787099:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.787101:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776914.787103:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776914/real 1775776914] req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776914.787109:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776914.787110:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776914.787111:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776914.787113:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776914.787114:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776914.787115:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776914.787117:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776914.787122:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787126:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776914.787127:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787128:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.787130:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776914.787133:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787139:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776914.787143:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787150:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776914.787152:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787154:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.787155:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787157:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776914.787159:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776914.787162:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776914.787166:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776914.787168:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776914.787173:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776914.787181:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810688:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776914.787185:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776914.787188:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787194:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776914.787195:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.787197:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776914.787199:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776914.787201:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776914.787203:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810816:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776914.787207:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776914.787209:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776914.787210:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.787212:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776914.787215:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776914.787217:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776914.787220:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776914.787222:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776914.787224:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776914.787226:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099810816, portal 4 00000100:00000001:0.0:1775776914.787227:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776914.787228:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751264256 : -114841958287360 : ffff978d45def800) 00000100:00000040:0.0:1775776914.787232:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787235:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776914.787237:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776914.787238:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776914.787240:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099810816, offset 0 00000400:00000200:0.0:1775776914.787242:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776914.787245:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.787247:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776914.787249:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776914.787250:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776914.787252:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776914.787253:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776914.787255:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4565:4565:256:4294967295] 192.168.201.139@tcp LPNI seq info [4565:4565:8:4294967295] 00000400:00000200:0.0:1775776914.787258:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.787261:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776914.787264:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.787265:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776914.787268:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776914.787269:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776914.787272:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776914.787273:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776914.787276:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.787277:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776914.787279:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776914.787281:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776914.787282:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776914.787284:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776914.787285:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776914.787287:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787291:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.787293:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787297:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.787298:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776914.787300:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776914.787302:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776914.787304:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776914.787305:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313302549 00000100:00000001:0.0:1775776914.787307:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313303798 00000100:00000001:0.0:1775776914.787308:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313302549 00000100:00000001:0.0:1775776914.787309:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313303798 00000100:00000001:0.0:1775776914.787311:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776914.787312:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.787313:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776914.787315:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776914/real 1775776914] req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776914.787319:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776914.787319:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776914.787320:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776914.787321:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776914.787322:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776914.787323:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776914.787324:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776914.787329:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787332:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776914.787333:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787334:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.787335:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776914.787337:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787340:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776914.787342:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787347:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776914.787348:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787349:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.787350:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787351:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776914.787352:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776914.787353:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776914.787355:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776914.787356:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776914.787358:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776914.787362:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810816:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776914.787364:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776914.787365:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787369:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776914.787369:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.787370:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776914.787371:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776914.787372:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776914.787374:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810944:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776914.787376:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776914.787377:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776914.787378:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776914.787379:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776914.787381:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776914.787382:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776914.787384:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776914.787386:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776914.787387:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776914.787389:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099810944, portal 4 00000100:00000001:0.0:1775776914.787390:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776914.787391:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751260672 : -114841958290944 : ffff978d45deea00) 00000100:00000040:0.0:1775776914.787394:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787397:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776914.787399:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776914.787400:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776914.787402:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099810944, offset 0 00000400:00000200:0.0:1775776914.787404:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776914.787406:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.787408:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776914.787409:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776914.787411:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776914.787412:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776914.787414:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776914.787416:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4566:4566:256:4294967295] 192.168.201.139@tcp LPNI seq info [4566:4566:8:4294967295] 00000400:00000200:0.0:1775776914.787418:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776914.787422:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776914.787424:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.787426:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776914.787427:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776914.787429:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776914.787433:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776914.787435:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776914.787438:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776914.787440:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776914.787443:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776914.787446:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776914.787448:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776914.787449:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776914.787452:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776914.787455:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787463:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.787466:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787472:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.787474:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776914.787477:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776914.787479:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776914.787482:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776914.787484:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313302540 00000100:00000001:0.0:1775776914.787486:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313303798 00000100:00000001:0.0:1775776914.787488:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313302540 00000100:00000001:0.0:1775776914.787489:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313303798 00000100:00000001:0.0:1775776914.787491:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776914.787492:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.787493:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776914.787496:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776914/real 1775776914] req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776914.787500:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776914.787501:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776914.787502:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776914.787503:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776914.787504:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776914.787506:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776914.787508:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776914.787512:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787520:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776914.787521:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787522:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.787523:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776914.787525:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787528:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776914.787531:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787535:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776914.787537:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787538:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776914.787538:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787539:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776914.787541:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776914.787542:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776914.787544:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776914.787546:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776914.787548:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776914.787552:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099810944:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776914.787554:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787555:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.787557:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedf80 x1862033099810688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787561:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776914.787563:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776914.787565:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775776914.787567:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776914.787568:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776914.787569:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776914.787571:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776914.787573:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158000. 02000000:00000001:0.0:1775776914.787575:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776914.787591:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776914.787593:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedf80. 00000100:00000001:0.0:1775776914.787597:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776914.787598:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787600:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.787606:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def800 x1862033099810816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787613:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776914.787615:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776914.787616:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776914.787618:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776914.787620:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776914.787621:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776914.787622:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776914.787624:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158800. 02000000:00000001:0.0:1775776914.787626:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776914.787627:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776914.787628:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def800. 00000100:00000001:0.0:1775776914.787630:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776914.787631:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787633:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776914.787636:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099810944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776939 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776914.787641:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776914.787642:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776914.787644:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776914.787646:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776914.787647:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776914.787648:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776914.787651:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776914.787652:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a000. 02000000:00000001:0.0:1775776914.787654:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776914.787655:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776914.787656:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:0.0:1775776914.787658:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776914.787659:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787662:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776914.787669:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776914.787670:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776914.787672:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776914.787673:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776915.484118:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776915.484125:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776915.484127:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776915.484132:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776915.484136:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776915.484139:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776915.484141:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776915.484143:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776915.484146:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776915.548488:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776915.548494:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776915.548499:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776915.548501:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776915.548504:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776915.548505:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776915.548509:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776915.548510:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776915.548694:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776915.548697:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776915.548701:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776915.677552:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776915.677591:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776915.677594:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776915.677600:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776915.677603:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776915.677607:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776915.677609:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776915.677611:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776915.677614:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776916.059921:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313303798, at 4313303808 00000100:00000010:1.0:1775776916.059941:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f2372a0. 00000100:00000040:0.0:1775776916.060002:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313303798, at 4313303808 00000100:00000010:0.0:1775776916.060011:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775776916.507906:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776916.507913:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776916.507914:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776916.507918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776916.507921:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776916.507924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776916.507926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776916.507928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776916.507931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776916.571897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776916.571902:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776916.571907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776916.571908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776916.571910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776916.571911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776916.571914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776916.571915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776916.572019:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776916.572022:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776916.572026:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776916.700496:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776916.700504:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776916.700506:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776916.700511:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776916.700515:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776916.700518:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776916.700521:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776916.700523:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776916.700526:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776917.532598:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776917.532605:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776917.532607:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776917.532613:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776917.532616:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776917.532620:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776917.532622:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776917.532625:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776917.532627:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776917.596136:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776917.596143:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776917.596149:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776917.596150:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776917.596154:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776917.596156:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776917.596159:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776917.596160:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776917.598821:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776917.598828:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776917.598832:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776917.724067:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776917.724074:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776917.724076:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776917.724083:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776917.724086:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776917.724090:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776917.724094:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776917.724096:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776917.724099:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776918.555994:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776918.556008:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776918.556011:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776918.556016:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776918.556020:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776918.556023:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776918.556026:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776918.556028:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776918.556031:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776918.619989:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776918.619998:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776918.620000:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776918.620003:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776918.620007:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776918.620010:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776918.620011:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776918.620013:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776918.620017:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776918.620018:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776918.620021:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776918.620023:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776918.620025:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776918.620026:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776918.622368:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776918.622374:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776918.622379:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776918.748122:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776918.748130:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776918.748132:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776918.748138:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776918.748142:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776918.748146:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776918.748148:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776918.748150:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776918.748153:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776919.579903:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776919.579911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776919.579913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776919.579919:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776919.579923:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776919.579926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776919.579929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776919.579931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776919.579933:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776919.644732:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776919.644738:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776919.644743:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776919.644744:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776919.644748:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776919.644749:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776919.644753:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776919.644754:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776919.646452:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776919.646456:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776919.646461:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776919.771895:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776919.771901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776919.771903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776919.771909:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776919.771912:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776919.771916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776919.771919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776919.771921:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776919.771924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776919.900145:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776919.900157:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776919.900160:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776919.900164:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776919.900168:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776919.900175:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18635 00000400:00000200:0.0:1775776919.900185:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776919.900191:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776919.900195:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776919.900198:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900202:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776919.900204:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900206:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776919.900211:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776919.900213:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776919.900216:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776919.900219:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776919.900221:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776919.900223:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776919.900226:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900228:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776919.900230:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900231:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776919.900234:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776919.900238:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900241:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776919.900243:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776919.900247:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776919.900250:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900252:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776919.900254:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776919.900263:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776919.900267:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776919.900270:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776919.900271:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900274:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776919.900277:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776919.900280:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776919.900287:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776919.900293:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776919.900301:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776919.900312:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776919.900317:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776919.900321:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776919.900348:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776919.900351:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776919.900354:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776919.900355:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776919.900357:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776919.900359:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776919.900362:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18635 00000400:00000200:0.0:1775776919.900366:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776919.900368:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776919.900370:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776919.900373:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900374:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776919.900376:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900378:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776919.900380:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776919.900381:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776919.900382:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776919.900384:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776919.900386:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776919.900388:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776919.900391:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900393:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776919.900395:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900396:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776919.900398:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776919.900401:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900404:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776919.900406:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776919.900411:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776919.900413:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900414:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776919.900418:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776919.900421:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900423:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776919.900426:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776919.900429:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776919.900431:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776919.900432:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900433:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776919.900435:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776919.900436:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776919.900440:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776919.900443:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776919.900448:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776919.900455:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776919.900457:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776919.900461:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776919.900468:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776919.900470:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776919.900474:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776919.900475:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776919.900477:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776919.900479:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776919.900481:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18635 00000400:00000200:0.0:1775776919.900485:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776919.900502:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776919.900504:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776919.900507:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900509:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776919.900511:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900513:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776919.900515:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776919.900516:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776919.900518:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776919.900520:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776919.900522:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776919.900524:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776919.900526:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900528:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776919.900530:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900531:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776919.900533:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776919.900536:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900538:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776919.900540:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776919.900542:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776919.900544:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900546:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776919.900548:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776919.900550:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900552:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776919.900555:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776919.900557:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776919.900560:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776919.900561:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900562:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776919.900564:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776919.900565:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776919.900568:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776919.900572:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776919.900576:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776919.900584:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776919.900590:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776919.900594:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776919.900602:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776919.900604:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776919.900608:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776919.900610:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776919.900612:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776919.900615:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776919.900617:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18635 00000400:00000200:0.0:1775776919.900621:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776919.900621:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1775776919.900626:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:2.0:1775776919.900627:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775776919.900628:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776919.900630:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776919.900632:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900635:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776919.900639:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000040:2.0:1775776919.900640:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776919.900642:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776919.900646:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776919.900648:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776919.900651:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776919.900654:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776919.900658:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776919.900658:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775776919.900662:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776919.900662:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776919.900667:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000001:2.0:1775776919.900668:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775776919.900670:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775776919.900672:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775776919.900674:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900677:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:2.0:1775776919.900678:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776919.900681:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00100000:2.0:1775776919.900685:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099811072:192.168.201.139@tcp:250:kworker.0 00000100:00080000:0.0:1775776919.900686:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900692:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776919.900694:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1775776919.900694:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776919.900698:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776919.900700:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776919.900701:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900704:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775776919.900704:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776919.900708:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 02000000:00000001:2.0:1775776919.900708:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1775776919.900713:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900714:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776919.900715:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504400. 00000100:00000010:0.0:1775776919.900718:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 02000000:00000001:2.0:1775776919.900719:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776919.900722:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776919.900726:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776919.900728:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.900729:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776919.900732:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776919.900734:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1775776919.900735:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 02000000:00000001:0.0:1775776919.900736:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776919.900742:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000400:00000010:2.0:1775776919.900744:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000001:0.0:1775776919.900746:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776919.900752:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775776919.900753:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099811072, portal 25 00000100:00000001:2.0:1775776919.900758:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776919.900761:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000100:00000001:0.0:1775776919.900768:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1775776919.900771:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776919.900772:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.900806:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776919.900810:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776919.900814:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776919.900818:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099811072, offset 0 00000100:00000040:0.0:1775776919.900821:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776919.900828:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900837:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.900841:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:0.0:1775776919.900842:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776919.900846:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:2.0:1775776919.900848:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:0.0:1775776919.900849:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776919.900852:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775776919.900855:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775776919.900856:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776919.900860:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18635 00000400:00000200:2.0:1775776919.900862:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776919.900868:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.900869:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776919.900874:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776919.900876:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776919.900879:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775776919.900883:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4567:4567:256:4294967295] 192.168.201.139@tcp LPNI seq info [4567:4567:8:4294967295] 00000100:00000040:0.0:1775776919.900885:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900888:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776919.900891:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776919.900893:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776919.900894:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776919.900898:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776919.900899:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776919.900901:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776919.900904:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776919.900908:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776919.900912:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000400:00000200:2.0:1775776919.900912:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000040:0.0:1775776919.900918:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900921:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1775776919.900925:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000040:0.0:1775776919.900926:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900929:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000800:00000010:2.0:1775776919.900931:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000020:00000040:0.0:1775776919.900932:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776919.900937:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776919.900939:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776919.900942:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1775776919.900943:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776919.900946:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1775776919.900948:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 02000000:00000001:0.0:1775776919.900949:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900953:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776919.900955:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000800:00000200:2.1:1775776919.900959:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00000001:0.0:1775776919.900960:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776919.900963:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776919.900964:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000100:00000010:0.0:1775776919.900968:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775776919.900971:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000800:00000100:2.0:1775776919.900973:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 02000000:00000001:0.0:1775776919.900974:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776919.900977:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776919.900977:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000100:00000001:0.0:1775776919.900980:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776919.900982:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776919.900984:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.900986:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:0.0:1775776919.900988:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776919.900993:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.900995:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00100000:0.0:1775776919.901000:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776919.901002:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776919.901006:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:0.0:1775776919.901011:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776919.901014:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775776919.901015:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775776919.901023:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776919.901024:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776919.901050:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.901050:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.901056:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776919.901057:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18645) 00000100:00000001:2.0:1775776919.901078:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.901080:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776919.901086:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.901090:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776919.901093:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313308918 00000100:00000010:2.0:1775776919.901272:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b4e0. 00000100:00000001:2.0:1775776919.901280:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776919.901286:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313308918 00000100:00000010:2.0:1775776919.901289:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b8a0. 00000100:00000001:2.0:1775776919.901292:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776919.901295:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776919.901298:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.901300:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776919.901305:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776919/real 1775776919] req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776919.901315:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776919.901316:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776919.901318:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776919.901320:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776919.901322:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776919.901324:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776919.901327:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776919.901334:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901340:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776919.901342:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901344:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.901346:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776919.901349:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901355:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776919.901359:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901366:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776919.901369:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901371:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.901372:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901374:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776919.901376:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776919.901379:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776919.901383:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776919.901385:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776919.901388:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776919.901395:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099811072:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776919.901400:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776919.901402:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901408:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776919.901409:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776919.901411:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776919.901413:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776919.901415:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776919.901417:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811200:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776919.901421:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776919.901423:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776919.901424:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776919.901426:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776919.901429:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776919.901431:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.901435:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776919.901437:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776919.901441:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776919.901443:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099811200, portal 10 00000100:00000001:2.0:1775776919.901445:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776919.901446:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:2.0:1775776919.901452:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901457:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776919.901459:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776919.901461:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776919.901463:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099811200, offset 0 00000400:00000200:2.0:1775776919.901467:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776919.901471:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.901475:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776919.901478:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776919.901481:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776919.901483:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776919.901485:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776919.901498:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4568:4568:256:4294967295] 192.168.201.139@tcp LPNI seq info [4568:4568:8:4294967295] 00000400:00000200:2.0:1775776919.901502:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.901508:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776919.901513:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776919.901516:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776919.901520:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776919.901523:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776919.901527:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776919.901529:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776919.901533:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776919.901535:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776919.901538:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776919.901541:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776919.901544:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776919.901545:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776919.901548:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776919.901551:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901558:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.901561:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901567:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.901568:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776919.901571:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776919.901575:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776919.901577:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776919.901579:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313307653 00000100:00000001:2.0:1775776919.901581:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313308918 00000100:00000001:2.0:1775776919.901583:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313307653 00000100:00000001:2.0:1775776919.901585:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313308918 00000100:00000001:2.0:1775776919.901587:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776919.901588:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.901590:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776919.901593:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776919/real 1775776919] req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776919.901601:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776919.901602:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776919.901604:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776919.901606:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776919.901608:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776919.901610:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776919.901613:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776919.901620:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901626:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776919.901628:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901629:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.901630:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776919.901633:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901640:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776919.901643:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901649:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776919.901651:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901653:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.901654:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901655:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776919.901657:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776919.901660:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776919.901663:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776919.901665:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776919.901668:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776919.901675:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811200:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776919.901679:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776919.901682:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901687:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776919.901689:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776919.901690:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776919.901692:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776919.901694:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776919.901697:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811328:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776919.901699:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776919.901701:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776919.901702:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776919.901704:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776919.901707:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776919.901709:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.901712:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776919.901714:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776919.901718:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776919.901720:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099811328, portal 10 00000100:00000001:2.0:1775776919.901722:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776919.901723:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:2.0:1775776919.901729:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901734:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776919.901736:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776919.901738:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776919.901741:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099811328, offset 0 00000400:00000200:2.0:1775776919.901745:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776919.901749:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.901751:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776919.901754:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776919.901757:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776919.901759:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776919.901761:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776919.901764:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4569:4569:256:4294967295] 192.168.201.139@tcp LPNI seq info [4569:4569:8:4294967295] 00000400:00000200:2.0:1775776919.901768:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.901773:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776919.901798:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776919.901801:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776919.901805:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776919.901808:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776919.901812:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776919.901814:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776919.901818:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776919.901819:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776919.901822:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776919.901826:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776919.901828:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776919.901830:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776919.901832:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776919.901835:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901842:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.901845:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901850:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.901852:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776919.901855:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776919.901857:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776919.901860:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776919.901862:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313307655 00000100:00000001:2.0:1775776919.901864:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313308918 00000100:00000001:2.0:1775776919.901866:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313307655 00000100:00000001:2.0:1775776919.901869:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313308918 00000100:00000001:2.0:1775776919.901871:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776919.901873:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.901874:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776919.901877:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776919/real 1775776919] req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776919.901884:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776919.901885:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776919.901887:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776919.901888:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776919.901890:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776919.901892:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776919.901895:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776919.901902:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901908:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776919.901909:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901911:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.901912:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776919.901915:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901921:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776919.901924:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901930:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776919.901932:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901933:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.901935:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901937:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776919.901938:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776919.901940:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776919.901943:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776919.901945:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776919.901949:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776919.901956:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811328:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776919.901960:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901962:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.901965:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099811072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.901971:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776919.901973:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776919.901976:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776919.901978:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776919.901980:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776919.901982:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776919.901984:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776919.901986:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:2.0:1775776919.901989:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776919.901991:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776919.901992:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:2.0:1775776919.901995:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776919.901996:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.901997:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.902000:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099811200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902007:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776919.902008:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776919.902009:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776919.902011:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776919.902012:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776919.902014:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776919.902015:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776919.902016:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775776919.902018:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776919.902019:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776919.902021:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000100:00000001:2.0:1775776919.902022:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776919.902024:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902025:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.902028:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099811328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902033:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776919.902034:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776919.902035:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776919.902037:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776919.902038:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776919.902039:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776919.902041:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776919.902042:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:2.0:1775776919.902044:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776919.902045:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776919.902046:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:2.0:1775776919.902048:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776919.902049:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902051:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902271:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776919.902274:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.902276:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776919.902278:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776919.902279:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776919.902284:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902290:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776919.902291:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776919.902294:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776919.902295:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776919.902297:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776919.902300:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811456:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776919.902303:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776919.902304:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776919.902306:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776919.902308:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776919.902311:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776919.902312:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.902316:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776919.902318:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776919.902322:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776919.902324:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099811456, portal 4 00000100:00000001:2.0:1775776919.902326:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776919.902327:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:2.0:1775776919.902332:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902337:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776919.902339:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776919.902341:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776919.902343:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099811456, offset 0 00000400:00000200:2.0:1775776919.902346:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776919.902350:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.902353:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776919.902356:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776919.902358:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776919.902360:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776919.902362:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776919.902365:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4570:4570:256:4294967295] 192.168.201.139@tcp LPNI seq info [4570:4570:8:4294967295] 00000400:00000200:2.0:1775776919.902368:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.902374:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776919.902378:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776919.902381:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776919.902384:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776919.902387:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776919.902390:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776919.902393:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776919.902396:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776919.902398:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776919.902400:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776919.902403:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776919.902406:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776919.902408:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776919.902410:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776919.902413:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902420:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.902422:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902428:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.902429:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776919.902432:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776919.902434:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776919.902437:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776919.902439:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313307664 00000100:00000001:2.0:1775776919.902441:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313308918 00000100:00000001:2.0:1775776919.902443:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313307664 00000100:00000001:2.0:1775776919.902445:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313308918 00000100:00000001:2.0:1775776919.902447:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776919.902448:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.902450:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776919.902453:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776919/real 1775776919] req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776919.902460:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776919.902461:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776919.902462:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776919.902465:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776919.902466:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776919.902468:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776919.902470:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776919.902477:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902483:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776919.902484:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902486:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.902501:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776919.902505:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902511:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776919.902514:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902521:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776919.902523:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902524:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.902525:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902527:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776919.902528:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776919.902531:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776919.902534:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776919.902536:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776919.902539:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776919.902547:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811456:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776919.902550:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776919.902553:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902559:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776919.902560:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776919.902562:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776919.902563:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776919.902565:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776919.902567:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811584:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776919.902570:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776919.902572:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776919.902574:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776919.902575:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776919.902578:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776919.902579:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776919.902582:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776919.902585:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776919.902587:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776919.902590:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099811584, portal 4 00000100:00000001:2.0:1775776919.902592:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776919.902593:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:2.0:1775776919.902597:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902603:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776919.902605:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776919.902606:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776919.902608:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099811584, offset 0 00000400:00000200:2.0:1775776919.902612:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776919.902616:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.902619:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776919.902621:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776919.902624:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776919.902626:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776919.902628:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776919.902631:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4571:4571:256:4294967295] 192.168.201.139@tcp LPNI seq info [4571:4571:8:4294967295] 00000400:00000200:2.0:1775776919.902635:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776919.902640:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776919.902644:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776919.902647:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776919.902650:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776919.902652:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776919.902656:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776919.902658:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776919.902661:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776919.902663:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776919.902665:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776919.902668:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776919.902671:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776919.902672:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776919.902675:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776919.902678:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902685:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.902687:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776944 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902693:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.902695:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776919.902698:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776919.902700:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776919.902703:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776919.902705:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313307655 00000100:00000001:2.0:1775776919.902708:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313308918 00000100:00000001:2.0:1775776919.902710:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313307655 00000100:00000001:2.0:1775776919.902712:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313308918 00000100:00000001:2.0:1775776919.902714:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776919.902716:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.902718:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776919.902721:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776919/real 1775776919] req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776919.902728:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776919.902730:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776919.902731:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776919.902734:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776919.902735:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776919.902737:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776919.902740:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776919.902748:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902754:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776919.902755:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902757:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.902758:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776919.902761:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902768:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776919.902771:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902777:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776919.902800:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902801:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776919.902802:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902804:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776919.902806:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776919.902808:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776919.902812:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776919.902814:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776919.902817:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776919.902825:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811584:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776919.902829:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902830:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.902833:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099811456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902839:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776919.902840:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776919.902842:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776919.902844:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776919.902845:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776919.902847:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776919.902849:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776919.902850:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:2.0:1775776919.902853:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776919.902854:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776919.902855:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:2.0:1775776919.902857:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776919.902858:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902859:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776919.902862:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099811584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776944 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776919.902873:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776919.902874:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776919.902875:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776919.902877:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776919.902878:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776919.902879:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776919.902881:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776919.902882:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:2.0:1775776919.902885:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776919.902886:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776919.902887:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:2.0:1775776919.902890:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776919.902891:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902893:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776919.902896:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776919.902897:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776919.902899:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776919.902900:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776920.603891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776920.603898:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776920.603901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776920.603907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776920.603910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776920.603914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776920.603917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776920.603920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776920.603922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668859:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776920.668865:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668871:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776920.668874:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776920.668876:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776920.668880:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668883:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776920.668886:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668888:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668891:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668894:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776920.668896:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776920.668897:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776920.668898:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668899:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776920.668900:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668902:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668903:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776920.668908:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776920.668909:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776920.668910:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668912:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776920.668913:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668914:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776920.668916:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776920.669017:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776920.669021:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776920.669023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776920.796026:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776920.796034:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776920.796036:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776920.796042:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776920.796046:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776920.796050:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776920.796052:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776920.796055:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776920.796057:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776921.197645:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313308918, at 4313308945 00000100:00000010:1.0:1775776921.197662:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000040:0.0:1775776921.197893:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313308918, at 4313308946 00000100:00000010:0.0:1775776921.197899:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000001:3.0:1775776921.628717:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776921.628724:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776921.628726:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776921.628731:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776921.628735:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776921.628739:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776921.628741:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776921.628743:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776921.628746:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776921.691910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776921.691918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776921.691924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776921.691925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776921.691928:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776921.691930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776921.691934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776921.691935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776921.692511:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776921.692515:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776921.692518:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776921.821347:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776921.821354:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776921.821356:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776921.821362:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776921.821365:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776921.821369:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776921.821372:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776921.821374:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776921.821377:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776922.652184:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776922.652190:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776922.652193:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776922.652198:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776922.652202:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776922.652205:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776922.652209:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776922.652211:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776922.652214:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776922.716580:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776922.716587:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776922.716593:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776922.716594:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776922.716598:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776922.716598:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776922.716601:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776922.716602:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776922.716671:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776922.716674:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776922.716677:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776922.843896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776922.843903:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776922.843905:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776922.843910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776922.843914:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776922.843918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776922.843920:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776922.843922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776922.843925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776923.676252:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776923.676260:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776923.676262:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776923.676267:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776923.676270:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776923.676274:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776923.676276:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776923.676279:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776923.676281:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776923.740073:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776923.740079:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776923.740084:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776923.740085:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776923.740088:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776923.740090:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776923.740093:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776923.740095:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776923.741536:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776923.741540:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776923.741544:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776923.869057:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776923.869064:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776923.869066:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776923.869072:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776923.869076:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776923.869079:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776923.869082:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776923.869084:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776923.869087:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776924.699891:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776924.699897:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776924.699899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776924.699904:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776924.699907:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776924.699911:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776924.699914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776924.699917:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776924.699920:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776924.763952:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776924.763958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776924.763963:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776924.763964:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776924.763968:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776924.763969:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776924.763972:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776924.763973:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776924.770488:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776924.770494:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776924.770498:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776924.891988:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776924.891995:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776924.891997:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776924.892003:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776924.892006:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776924.892010:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776924.892012:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776924.892015:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776924.892018:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776925.019901:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776925.019912:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776925.019916:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776925.019920:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776925.019924:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776925.019932:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18640 00000400:00000200:0.0:1775776925.019942:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776925.019949:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776925.019952:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776925.019955:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.019959:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776925.019961:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.019964:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776925.019967:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776925.019969:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776925.019972:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776925.019974:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776925.019977:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776925.019979:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776925.019982:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.019984:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776925.019986:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.019988:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776925.019991:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776925.019994:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776925.019997:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776925.020000:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776925.020004:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776925.020007:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.020010:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776925.020012:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776925.020023:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775776925.020026:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776925.020030:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776925.020032:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.020035:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776925.020037:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776925.020041:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776925.020050:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775776925.020057:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776925.020065:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776925.020076:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776925.020081:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776925.020087:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776925.020110:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776925.020113:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776925.020116:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776925.020118:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776925.020120:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776925.020123:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776925.020126:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18640 00000400:00000200:0.0:1775776925.020130:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776925.020134:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776925.020136:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776925.020137:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776925.020141:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776925.020142:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776925.020145:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776925.020146:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776925.020150:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.020153:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776925.020156:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:2.0:1775776925.020156:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776925.020159:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776925.020163:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776925.020167:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776925.020172:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776925.020173:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775776925.020176:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776925.020176:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.020181:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775776925.020184:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000001:2.0:1775776925.020186:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776925.020189:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775776925.020190:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776925.020194:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.020197:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00100000:2.0:1775776925.020197:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099811712:192.168.201.139@tcp:250:kworker.0 00000020:00000040:0.0:1775776925.020204:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000001:2.0:1775776925.020205:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1775776925.020211:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 02000000:00000001:2.0:1775776925.020211:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776925.020214:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.020215:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776925.020218:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776925.020218:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776925.020225:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:0.0:1775776925.020226:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776925.020229:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776925.020230:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.020234:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775776925.020238:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000008:00100000:0.0:1775776925.020239:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776925.020243:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776925.020243:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000100:00000001:0.0:1775776925.020248:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776925.020253:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000400:00000010:2.0:1775776925.020253:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000020:00000040:0.0:1775776925.020258:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776925.020261:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:2.0:1775776925.020262:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099811712, portal 25 02000000:00000001:0.0:1775776925.020263:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.020268:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1775776925.020268:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1775776925.020272:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1775776925.020272:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757056 : -114841967794560 : ffff978d454de680) 02000000:00000001:0.0:1775776925.020275:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776925.020280:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775776925.020281:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d000. 00000100:00000001:0.0:1775776925.020287:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.020293:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1775776925.020297:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775776925.020299:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776925.020303:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776925.020309:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099811712, offset 0 00000100:00000001:0.0:1775776925.020312:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776925.020316:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.020319:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000040:0.0:1775776925.020323:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776925.020330:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.020338:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:0.0:1775776925.020339:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.020344:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776925.020348:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776925.020351:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776925.020354:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776925.020358:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4572:4572:256:4294967295] 192.168.201.139@tcp LPNI seq info [4572:4572:8:4294967295] 00000400:00000200:2.0:1775776925.020363:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.020372:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776925.020378:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.020383:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776925.020389:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776925.020392:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776925.020417:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776925.020420:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776925.020425:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.020427:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776925.020434:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776925.020438:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776925.020443:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776925.020446:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776925.020449:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776925.020453:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020463:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.020466:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020473:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.020475:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776925.020480:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.020484:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776925.020487:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313314037 00000100:00000010:2.0:1775776925.020491:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b3c0. 00000100:00000001:2.0:1775776925.020497:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776925.020582:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313314037 00000100:00000010:2.0:1775776925.020587:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b4e0. 00000100:00000001:2.0:1775776925.020590:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776925.020594:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776925.020597:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.020599:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776925.020603:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776925/real 1775776925] req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776925.020613:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776925.020614:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776925.020616:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776925.020619:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776925.020620:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776925.020623:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776925.020626:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776925.020634:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020640:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776925.020642:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.020644:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.020646:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776925.020650:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020657:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776925.020661:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020668:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776925.020670:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.020672:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.020673:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.020675:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776925.020677:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776925.020680:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776925.020684:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776925.020686:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776925.020690:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776925.020697:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099811712:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776925.020702:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.020704:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.020706:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de680 x1862033099811712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020713:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776925.020715:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776925.020726:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776925.020729:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776925.020731:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776925.020732:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776925.020734:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776925.020737:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759dc00. 02000000:00000001:2.0:1775776925.020740:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776925.020742:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776925.020744:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de680. 00000100:00000001:2.0:1775776925.020747:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776925.020748:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.020750:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.020756:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776925.020758:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.020760:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776925.020761:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776925.020762:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776925.020797:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020804:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776925.020806:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.020808:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776925.020810:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776925.020811:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776925.020814:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811840:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776925.020817:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776925.020819:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776925.020821:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.020822:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776925.020825:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776925.020827:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.020830:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776925.020833:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776925.020836:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776925.020839:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099811840, portal 10 00000100:00000001:2.0:1775776925.020841:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776925.020842:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741762432 : -114841967789184 : ffff978d454dfb80) 00000100:00000040:2.0:1775776925.020848:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020854:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776925.020856:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776925.020858:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776925.020861:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099811840, offset 0 00000400:00000200:2.0:1775776925.020864:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776925.020868:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.020871:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776925.020874:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776925.020888:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776925.020891:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776925.020893:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776925.020896:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4573:4573:256:4294967295] 192.168.201.139@tcp LPNI seq info [4573:4573:8:4294967295] 00000400:00000200:2.0:1775776925.020901:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.020908:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776925.020912:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.020915:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776925.020919:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776925.020922:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776925.020925:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776925.020928:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776925.020931:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.020933:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776925.020936:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776925.020939:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776925.020949:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776925.020951:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776925.020953:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776925.020956:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020964:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.020967:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.020974:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.020975:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776925.020978:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776925.020981:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776925.020984:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776925.020986:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313312773 00000100:00000001:2.0:1775776925.020988:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313314037 00000100:00000001:2.0:1775776925.020990:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313312773 00000100:00000001:2.0:1775776925.020992:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313314037 00000100:00000001:2.0:1775776925.020994:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776925.020996:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.020998:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776925.021001:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776925/real 1775776925] req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776925.021008:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776925.021010:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776925.021011:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776925.021013:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776925.021014:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776925.021016:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776925.021019:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776925.021027:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.021033:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776925.021035:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.021036:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.021037:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776925.021040:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.021047:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776925.021050:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.021057:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776925.021058:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.021060:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.021061:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.021063:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776925.021064:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776925.021067:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776925.021070:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776925.021072:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776925.021076:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776925.021083:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811840:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776925.021087:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.021088:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.021091:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dfb80 x1862033099811840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.021097:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776925.021098:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776925.021100:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776925.021102:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776925.021104:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776925.021105:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776925.021107:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776925.021108:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d000. 02000000:00000001:2.0:1775776925.021111:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776925.021112:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776925.021113:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dfb80. 00000100:00000001:2.0:1775776925.021115:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776925.021117:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.021119:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.021122:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776925.021123:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.021125:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776925.021126:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776925.021427:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776925.021430:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776925.021432:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776925.021434:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776925.021437:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776925.021440:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18640 00000400:00000200:0.0:1775776925.021445:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776925.021448:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776925.021450:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776925.021454:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021456:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776925.021458:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021460:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776925.021462:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776925.021463:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776925.021465:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776925.021467:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776925.021468:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776925.021470:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776925.021473:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021474:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776925.021476:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021477:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776925.021479:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776925.021482:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021484:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776925.021485:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776925.021488:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776925.021490:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.021492:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776925.021494:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776925.021496:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.021498:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776925.021501:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775776925.021503:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776925.021505:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776925.021506:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.021508:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776925.021509:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776925.021511:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776925.021515:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776925.021518:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776925.021523:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776925.021530:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776925.021533:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776925.021536:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776925.021553:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776925.021556:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776925.021559:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776925.021560:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776925.021561:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776925.021563:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776925.021566:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18640 00000400:00000200:0.0:1775776925.021570:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776925.021573:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776925.021575:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776925.021578:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021581:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776925.021581:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776925.021585:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776925.021585:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776925.021588:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776925.021589:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776925.021592:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776925.021595:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000100:00000040:2.0:1775776925.021595:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776925.021598:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776925.021602:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776925.021606:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776925.021611:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776925.021619:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021622:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776925.021624:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021626:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776925.021629:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000001:2.0:1775776925.021634:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775776925.021635:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776925.021636:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.021640:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.021640:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776925.021643:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776925.021644:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776925.021647:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776925.021648:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1775776925.021652:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811968:192.168.201.139@tcp:38:kworker.0 02000000:00000001:0.0:1775776925.021653:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.021656:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776925.021659:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1775776925.021660:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1775776925.021664:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.021664:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776925.021667:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.021667:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.021671:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1775776925.021674:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 02000000:00000010:2.0:1775776925.021676:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 00000020:00000040:0.0:1775776925.021677:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776925.021679:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776925.021683:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776925.021685:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.021686:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776925.021689:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776925.021691:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1775776925.021691:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 02000000:00000001:0.0:1775776925.021695:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776925.021697:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 02000000:00000010:0.0:1775776925.021701:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000200:2.0:1775776925.021702:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099811968, portal 10 00000100:00000001:0.0:1775776925.021705:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.021706:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776925.021708:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741762432 : -114841967789184 : ffff978d454dfb80) 00000100:00100000:0.0:1775776925.021713:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776925.021721:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776925.021723:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776925.021728:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776925.021735:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776925.021737:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776925.021740:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776925.021742:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776925.021743:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776925.021745:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776925.021746:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18640 00000400:00000200:0.0:1775776925.021750:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776925.021752:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776925.021754:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776925.021757:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021758:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776925.021760:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021762:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776925.021764:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776925.021765:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776925.021766:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776925.021768:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776925.021769:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776925.021771:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776925.021773:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021774:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776925.021776:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021795:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776925.021798:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776925.021801:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776925.021803:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776925.021805:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776925.021808:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776925.021811:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776925.021811:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776925.021815:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776925.021818:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776925.021824:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.021826:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776925.021827:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776925.021832:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000100:00000040:2.0:1775776925.021832:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000020:00000040:0.0:1775776925.021835:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000400:00000010:2.0:1775776925.021836:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 02000000:00000001:0.0:1775776925.021838:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:2.0:1775776925.021840:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099811968, offset 0 02000000:00000001:0.0:1775776925.021841:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776925.021845:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776925.021847:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776925.021850:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.021850:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 02000000:00000010:0.0:1775776925.021856:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000400:00000200:2.0:1775776925.021857:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776925.021862:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.021864:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776925.021869:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:0.0:1775776925.021870:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1775776925.021873:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000001:0.0:1775776925.021881:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776925.021883:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776925.021888:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776925.021895:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776925.021899:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18650) 00000400:00000200:2.0:1775776925.022006:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776925.022009:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776925.022012:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4574:4574:256:4294967295] 192.168.201.139@tcp LPNI seq info [4574:4574:8:4294967295] 00000400:00000200:2.0:1775776925.022017:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.022024:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776925.022029:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.022033:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776925.022037:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776925.022039:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776925.022044:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776925.022046:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776925.022050:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.022052:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776925.022055:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776925.022058:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776925.022061:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776925.022063:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776925.022066:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776925.022069:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022077:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.022079:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022085:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.022087:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776925.022090:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776925.022092:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776925.022095:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776925.022097:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313312775 00000100:00000001:2.0:1775776925.022099:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313314037 00000100:00000001:2.0:1775776925.022101:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313312775 00000100:00000001:2.0:1775776925.022103:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313314037 00000100:00000001:2.0:1775776925.022105:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776925.022108:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.022110:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776925.022113:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776925/real 1775776925] req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776925.022121:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776925.022122:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776925.022124:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776925.022126:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776925.022127:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776925.022129:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776925.022132:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776925.022140:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022146:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776925.022147:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022149:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.022150:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776925.022154:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022168:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776925.022172:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022179:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776925.022181:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022183:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.022184:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022185:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776925.022187:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776925.022190:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776925.022193:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776925.022195:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776925.022199:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776925.022206:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099811968:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776925.022210:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022212:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.022214:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dfb80 x1862033099811968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022220:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776925.022222:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776925.022224:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776925.022226:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776925.022227:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776925.022229:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776925.022230:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776925.022232:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:2.0:1775776925.022234:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776925.022235:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776925.022237:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dfb80. 00000100:00000001:2.0:1775776925.022239:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776925.022240:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022242:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022245:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776925.022247:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.022249:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776925.022250:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776925.022251:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776925.022254:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022261:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776925.022262:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.022264:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776925.022266:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776925.022268:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776925.022271:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812096:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776925.022274:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776925.022275:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776925.022277:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.022278:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776925.022281:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776925.022283:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.022286:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776925.022288:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776925.022291:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776925.022294:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099812096, portal 4 00000100:00000001:2.0:1775776925.022296:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776925.022297:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757056 : -114841967794560 : ffff978d454de680) 00000100:00000040:2.0:1775776925.022302:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022309:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776925.022311:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776925.022312:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776925.022315:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099812096, offset 0 00000400:00000200:2.0:1775776925.022318:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776925.022322:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.022325:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776925.022327:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776925.022330:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776925.022332:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776925.022334:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776925.022338:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4575:4575:256:4294967295] 192.168.201.139@tcp LPNI seq info [4575:4575:8:4294967295] 00000400:00000200:2.0:1775776925.022341:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.022347:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776925.022352:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.022354:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776925.022358:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776925.022360:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776925.022364:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776925.022367:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776925.022371:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.022373:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776925.022375:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776925.022378:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776925.022381:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776925.022382:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776925.022384:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776925.022387:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022481:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.022485:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022492:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.022494:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776925.022498:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776925.022500:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776925.022512:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776925.022514:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313312784 00000100:00000001:2.0:1775776925.022516:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313314037 00000100:00000001:2.0:1775776925.022519:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313312784 00000100:00000001:2.0:1775776925.022520:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313314037 00000100:00000001:2.0:1775776925.022522:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776925.022524:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.022526:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776925.022529:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776925/real 1775776925] req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776925.022538:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776925.022539:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776925.022540:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776925.022543:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776925.022544:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776925.022546:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776925.022549:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776925.022556:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022563:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776925.022564:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022566:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.022567:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776925.022571:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022577:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776925.022580:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022586:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776925.022588:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022590:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.022591:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.022593:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776925.022594:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776925.022596:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776925.022599:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776925.022601:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776925.022604:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776925.022612:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812096:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776925.022615:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776925.022618:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022624:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776925.022626:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.022628:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776925.022629:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776925.022631:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776925.022634:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812224:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776925.022637:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776925.022638:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776925.022640:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776925.022641:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776925.022644:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776925.022646:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776925.022650:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776925.022653:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776925.022655:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776925.022658:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099812224, portal 4 00000100:00000001:2.0:1775776925.022660:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776925.022662:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:2.0:1775776925.022668:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.022674:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776925.022676:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776925.022678:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776925.022680:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099812224, offset 0 00000400:00000200:2.0:1775776925.022685:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776925.022688:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.022691:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776925.022693:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776925.022696:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776925.022698:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776925.022700:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776925.022703:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4576:4576:256:4294967295] 192.168.201.139@tcp LPNI seq info [4576:4576:8:4294967295] 00000400:00000200:2.0:1775776925.022707:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776925.022713:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776925.022849:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.022853:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776925.022857:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776925.022860:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1775776925.022864:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776925.022866:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1775776925.022870:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776925.022872:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776925.022874:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776925.023013:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776925.023017:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776925.023019:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776925.023022:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776925.023025:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.023033:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.023035:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.023042:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.023043:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776925.023046:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776925.023049:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776925.023052:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776925.023054:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313312776 00000100:00000001:2.0:1775776925.023057:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313314037 00000100:00000001:2.0:1775776925.023059:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313312776 00000100:00000001:2.0:1775776925.023061:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313314037 00000100:00000001:2.0:1775776925.023063:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776925.023064:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.023066:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776925.023069:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776925/real 1775776925] req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776925.023075:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776925.023077:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776925.023078:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776925.023080:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776925.023082:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776925.023084:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776925.023087:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776925.023095:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.023101:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776925.023102:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.023104:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.023105:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776925.023109:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.023116:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776925.023119:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.023125:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776925.023128:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.023129:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776925.023131:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.023132:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776925.023133:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776925.023136:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776925.023139:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776925.023141:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776925.023145:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776925.023152:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812224:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776925.023155:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.023157:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.023159:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de680 x1862033099812096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.023166:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776925.023167:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776925.023169:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776925.023171:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776925.023172:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776925.023174:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776925.023175:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776925.023177:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:2.0:1775776925.023179:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776925.023180:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776925.023182:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de680. 00000100:00000001:2.0:1775776925.023184:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776925.023185:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.023187:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776925.023190:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099812224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776925.023201:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776925.023202:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776925.023204:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776925.023206:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776925.023207:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776925.023208:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776925.023209:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776925.023210:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:2.0:1775776925.023212:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776925.023213:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776925.023214:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:2.0:1775776925.023216:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776925.023217:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.023218:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776925.023222:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776925.023223:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776925.023225:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776925.023226:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776925.724329:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776925.724335:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776925.724337:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776925.724342:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776925.724346:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776925.724349:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776925.724352:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776925.724354:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776925.724357:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776925.787903:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776925.787910:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776925.787915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776925.787917:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776925.787920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776925.787921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776925.787924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776925.787925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776925.788038:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776925.788042:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776925.788045:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776925.918665:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776925.918672:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776925.918674:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776925.918679:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776925.918682:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776925.918686:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776925.918689:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776925.918691:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776925.918694:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776926.299897:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313314037, at 4313314048 00000100:00000040:0.0:1775776926.299913:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313314037, at 4313314048 00000100:00000010:1.0:1775776926.299913:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000010:0.0:1775776926.299922:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000001:3.0:1775776926.748317:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776926.748324:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776926.748326:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776926.748331:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776926.748335:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776926.748338:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776926.748341:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776926.748343:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776926.748345:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776926.811922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776926.811926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776926.811931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776926.811932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776926.811934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776926.811935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776926.811937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776926.811938:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776926.813659:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776926.813664:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776926.813668:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776926.940917:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776926.940924:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776926.940926:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776926.940943:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776926.940947:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776926.940951:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776926.940954:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776926.940956:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776926.940959:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776927.771849:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776927.771855:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776927.771856:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776927.771862:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776927.771865:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776927.771869:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776927.771871:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776927.771873:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776927.771876:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776927.835864:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776927.835870:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776927.835876:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776927.835877:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776927.835880:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776927.835881:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776927.835884:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776927.835885:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776927.835948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776927.835952:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776927.835955:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776927.963881:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776927.963886:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776927.963888:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776927.963894:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776927.963898:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776927.963901:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776927.963904:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776927.963907:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776927.963909:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776928.795895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776928.795900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776928.795902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776928.795907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776928.795911:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776928.795915:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776928.795917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776928.795919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776928.795922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776928.859901:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776928.859909:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776928.859911:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776928.859923:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776928.859926:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776928.859929:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776928.859931:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776928.859933:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776928.859937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776928.859939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776928.859942:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776928.859943:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776928.859946:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776928.859947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776928.860006:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776928.860010:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776928.860012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776928.987874:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776928.987881:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776928.987883:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776928.987890:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776928.987893:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776928.987898:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776928.987900:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776928.987903:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776928.987906:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776929.819895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776929.819902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776929.819904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776929.819909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776929.819913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776929.819916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776929.819919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776929.819921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776929.819924:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776929.883876:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776929.883882:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776929.883888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776929.883889:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776929.883892:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776929.883893:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776929.883896:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776929.883897:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776929.884008:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776929.884011:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776929.884014:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776930.011897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776930.011904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776930.011906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776930.011911:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776930.011915:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776930.011918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776930.011921:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776930.011923:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776930.011926:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776930.139904:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776930.139915:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776930.139918:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776930.139922:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776930.139925:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776930.139932:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18645 00000400:00000200:3.0:1775776930.139941:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776930.139947:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776930.139950:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776930.139964:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.139968:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.139971:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.139973:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776930.139977:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776930.139978:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776930.139981:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776930.139984:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776930.139986:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776930.139988:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776930.139990:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.139992:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.139993:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.139995:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776930.139997:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776930.140000:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140003:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776930.140005:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776930.140008:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776930.140011:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140013:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776930.140015:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776930.140024:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775776930.140027:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776930.140030:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776930.140032:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140034:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776930.140036:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776930.140040:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776930.140050:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775776930.140056:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776930.140064:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776930.140073:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776930.140077:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776930.140081:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776930.140106:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776930.140109:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776930.140112:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776930.140113:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776930.140115:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776930.140117:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776930.140119:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18645 00000400:00000200:3.0:1775776930.140122:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776930.140125:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776930.140127:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776930.140129:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140131:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.140132:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140134:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776930.140136:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776930.140137:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776930.140138:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776930.140140:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776930.140142:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776930.140144:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776930.140146:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140147:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.140149:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140150:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776930.140152:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776930.140154:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140157:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776930.140158:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776930.140163:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776930.140164:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140166:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776930.140170:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776930.140173:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140174:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776930.140177:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775776930.140179:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776930.140182:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776930.140183:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140184:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776930.140185:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776930.140186:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776930.140189:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775776930.140192:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776930.140196:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776930.140202:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776930.140204:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776930.140207:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776930.140214:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776930.140216:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776930.140219:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776930.140220:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776930.140221:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776930.140224:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776930.140226:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18645 00000400:00000200:3.0:1775776930.140229:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776930.140231:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776930.140233:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776930.140246:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140248:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.140250:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140251:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776930.140253:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776930.140254:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776930.140255:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776930.140257:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776930.140258:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776930.140260:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775776930.140263:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140264:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.140266:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140267:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776930.140269:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776930.140271:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140272:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776930.140274:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776930.140276:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776930.140278:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140279:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776930.140281:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776930.140283:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140284:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776930.140287:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775776930.140289:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776930.140291:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776930.140292:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140293:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776930.140295:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776930.140296:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776930.140299:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775776930.140301:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776930.140305:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776930.140321:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776930.140322:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776930.140326:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776930.140333:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776930.140335:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776930.140338:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776930.140339:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776930.140340:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776930.140342:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776930.140344:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18645 00000400:00000200:3.0:1775776930.140347:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776930.140350:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776930.140352:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776930.140354:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140356:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.140358:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140360:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776930.140361:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776930.140362:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775776930.140363:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776930.140365:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776930.140367:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776930.140368:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775776930.140371:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140373:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.140375:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140376:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776930.140378:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776930.140381:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140383:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776930.140384:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776930.140386:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776930.140388:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140390:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776930.140392:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776930.140394:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140395:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776930.140398:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775776930.140400:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776930.140402:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776930.140403:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140405:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776930.140406:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776930.140407:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776930.140410:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775776930.140413:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776930.140416:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776930.140422:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776930.140424:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776930.140427:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776930.140433:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776930.140435:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776930.140438:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776930.140439:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776930.140440:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776930.140442:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776930.140444:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18645 00000400:00000200:3.0:1775776930.140448:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776930.140450:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776930.140452:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776930.140454:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140456:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.140457:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140459:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776930.140461:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776930.140462:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776930.140463:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776930.140465:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776930.140466:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776930.140468:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776930.140470:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140472:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776930.140474:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140475:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776930.140477:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776930.140479:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776930.140481:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776930.140483:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776930.140485:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776930.140493:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140494:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776930.140496:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776930.140498:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140500:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776930.140502:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775776930.140504:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776930.140506:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776930.140507:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.140509:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776930.140510:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776930.140511:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776930.140514:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775776930.140517:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776930.140520:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776930.140526:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776930.140528:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776930.140531:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776930.140537:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776930.140541:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18655) 00000100:00000001:2.0:1775776930.140747:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776930.140751:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776930.140753:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776930.140758:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.140765:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776930.140767:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.140770:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776930.140772:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776930.140775:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776930.140802:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099812352:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776930.140806:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776930.140810:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776930.140813:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.140815:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776930.140820:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776930.140821:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776930.140826:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776930.140830:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776930.140836:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776930.140842:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099812352, portal 25 00000100:00000001:2.0:1775776930.140845:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776930.140846:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751254400 : -114841958297216 : ffff978d45ded180) 00000100:00000040:2.0:1775776930.140851:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.140857:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776930.140859:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776930.140861:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776930.140865:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099812352, offset 0 00000400:00000200:2.0:1775776930.140871:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776930.140877:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.140882:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776930.140886:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776930.140890:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776930.140893:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776930.140896:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776930.140899:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4577:4577:256:4294967295] 192.168.201.139@tcp LPNI seq info [4577:4577:8:4294967295] 00000400:00000200:2.0:1775776930.140903:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.140912:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776930.140917:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.140922:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776930.140928:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776930.140931:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776930.140936:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776930.140939:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776930.140944:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.140946:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776930.140964:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776930.140969:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776930.140973:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776930.140975:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776930.140978:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776930.140982:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.140990:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.140993:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.140999:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.141001:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776930.141006:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776930.141009:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776930.141011:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313319158 00000100:00000010:2.0:1775776930.141015:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b8a0. 00000100:00000001:2.0:1775776930.141021:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776930.141026:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313319158 00000100:00000010:2.0:1775776930.141028:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b7e0. 00000100:00000001:2.0:1775776930.141031:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776930.141033:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776930.141036:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.141038:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776930.141041:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776930/real 1775776930] req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776930.141049:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776930.141051:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776930.141053:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776930.141055:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776930.141056:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776930.141058:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776930.141061:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776930.141069:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141074:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776930.141076:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141078:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.141080:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776930.141083:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141090:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776930.141094:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141100:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776930.141102:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141104:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.141105:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141107:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776930.141109:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776930.141111:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776930.141114:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776930.141116:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776930.141119:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776930.141127:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099812352:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776930.141131:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776930.141134:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141140:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776930.141141:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.141143:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776930.141144:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776930.141146:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776930.141148:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812480:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776930.141151:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776930.141153:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776930.141154:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.141156:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776930.141158:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776930.141160:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776930.141163:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776930.141165:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776930.141168:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776930.141170:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099812480, portal 10 00000100:00000001:2.0:1775776930.141172:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776930.141173:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751258880 : -114841958292736 : ffff978d45dee300) 00000100:00000040:2.0:1775776930.141178:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141183:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776930.141185:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776930.141187:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776930.141189:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099812480, offset 0 00000400:00000200:2.0:1775776930.141192:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776930.141196:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.141198:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776930.141200:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776930.141202:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776930.141204:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776930.141206:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776930.141209:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4578:4578:256:4294967295] 192.168.201.139@tcp LPNI seq info [4578:4578:8:4294967295] 00000400:00000200:2.0:1775776930.141212:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.141217:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776930.141221:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.141224:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776930.141227:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776930.141230:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776930.141233:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776930.141235:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776930.141239:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.141240:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776930.141243:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776930.141246:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776930.141248:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776930.141250:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776930.141252:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776930.141255:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141261:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.141264:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141270:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.141271:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776930.141274:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776930.141277:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776930.141280:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776930.141281:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313317893 00000100:00000001:2.0:1775776930.141284:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313319158 00000100:00000001:2.0:1775776930.141286:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313317893 00000100:00000001:2.0:1775776930.141288:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313319158 00000100:00000001:2.0:1775776930.141290:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776930.141291:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.141293:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776930.141296:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776930/real 1775776930] req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776930.141303:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776930.141304:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776930.141305:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776930.141307:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776930.141320:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776930.141323:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776930.141325:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776930.141333:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141339:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776930.141340:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141342:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.141343:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776930.141346:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141352:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776930.141355:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141361:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776930.141363:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141364:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.141365:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141367:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776930.141369:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776930.141371:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776930.141374:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776930.141376:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776930.141380:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776930.141387:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812480:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776930.141390:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776930.141393:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141399:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776930.141400:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.141402:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776930.141403:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776930.141405:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776930.141408:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812608:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776930.141411:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776930.141413:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776930.141414:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.141416:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776930.141419:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776930.141420:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776930.141423:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776930.141425:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776930.141428:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776930.141430:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099812608, portal 10 00000100:00000001:2.0:1775776930.141432:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776930.141433:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751255296 : -114841958296320 : ffff978d45ded500) 00000100:00000040:2.0:1775776930.141438:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141443:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776930.141445:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776930.141447:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776930.141449:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099812608, offset 0 00000400:00000200:2.0:1775776930.141453:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776930.141457:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.141460:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776930.141462:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776930.141465:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776930.141467:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776930.141469:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776930.141471:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4579:4579:256:4294967295] 192.168.201.139@tcp LPNI seq info [4579:4579:8:4294967295] 00000400:00000200:2.0:1775776930.141475:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.141481:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776930.141485:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.141488:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776930.141492:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776930.141495:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776930.141499:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776930.141501:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776930.141505:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.141507:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776930.141509:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776930.141512:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776930.141515:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776930.141517:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776930.141520:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776930.141523:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141529:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.141532:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141544:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.141546:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776930.141548:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776930.141551:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776930.141554:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776930.141555:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313317894 00000100:00000001:2.0:1775776930.141557:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313319158 00000100:00000001:2.0:1775776930.141559:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313317894 00000100:00000001:2.0:1775776930.141561:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313319158 00000100:00000001:2.0:1775776930.141563:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776930.141564:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.141566:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776930.141570:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776930/real 1775776930] req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776930.141577:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776930.141578:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776930.141580:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776930.141582:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776930.141583:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776930.141585:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776930.141587:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776930.141594:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141599:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776930.141600:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141601:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.141603:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776930.141606:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141611:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776930.141615:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141620:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776930.141622:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141623:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.141624:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141625:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776930.141627:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776930.141628:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776930.141631:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776930.141632:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776930.141636:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776930.141642:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812608:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776930.141646:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776930.141649:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141654:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776930.141655:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.141657:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776930.141658:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776930.141660:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776930.141663:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812736:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776930.141666:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776930.141667:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776930.141669:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.141671:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776930.141674:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776930.141675:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776930.141678:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776930.141680:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776930.141684:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776930.141686:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099812736, portal 4 00000100:00000001:2.0:1775776930.141688:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776930.141689:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751249920 : -114841958301696 : ffff978d45dec000) 00000100:00000040:2.0:1775776930.141694:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141699:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776930.141701:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776930.141703:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776930.141705:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099812736, offset 0 00000400:00000200:2.0:1775776930.141709:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776930.141712:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.141715:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776930.141717:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776930.141720:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776930.141722:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776930.141724:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776930.141726:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4580:4580:256:4294967295] 192.168.201.139@tcp LPNI seq info [4580:4580:8:4294967295] 00000400:00000200:2.0:1775776930.141730:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.141735:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776930.141739:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.141741:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776930.141745:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776930.141747:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776930.141750:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776930.141753:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776930.141756:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.141758:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776930.141760:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776930.141764:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776930.141766:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776930.141768:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776930.141770:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776930.141772:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141798:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.141801:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141808:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.141809:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776930.141811:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776930.141814:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776930.141817:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776930.141819:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313317904 00000100:00000001:2.0:1775776930.141821:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313319158 00000100:00000001:2.0:1775776930.141822:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313317904 00000100:00000001:2.0:1775776930.141824:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313319158 00000100:00000001:2.0:1775776930.141826:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776930.141828:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.141829:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776930.141832:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776930/real 1775776930] req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776930.141839:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776930.141840:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776930.141842:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776930.141844:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776930.141845:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776930.141848:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776930.141851:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776930.141858:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141863:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776930.141864:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141866:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.141867:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776930.141870:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141876:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776930.141879:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141885:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776930.141887:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141889:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.141890:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.141892:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776930.141893:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776930.141895:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776930.141898:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776930.141900:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776930.141903:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776930.141910:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812736:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776930.141914:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776930.141917:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141923:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776930.141924:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.141926:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776930.141927:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776930.141929:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776930.141932:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812864:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776930.141934:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776930.141936:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776930.141937:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776930.141939:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776930.141941:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776930.141943:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776930.141946:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776930.141948:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776930.141951:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776930.141953:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099812864, portal 4 00000100:00000001:2.0:1775776930.141955:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776930.141956:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751250816 : -114841958300800 : ffff978d45dec380) 00000100:00000040:2.0:1775776930.141960:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.141966:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776930.141967:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776930.141969:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776930.141972:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099812864, offset 0 00000400:00000200:2.0:1775776930.141975:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776930.141978:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.141981:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776930.141983:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776930.141986:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776930.141988:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776930.141989:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776930.141992:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4581:4581:256:4294967295] 192.168.201.139@tcp LPNI seq info [4581:4581:8:4294967295] 00000400:00000200:2.0:1775776930.141996:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776930.142001:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776930.142004:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.142007:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776930.142011:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776930.142013:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1775776930.142017:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776930.142019:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1775776930.142022:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776930.142024:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776930.142026:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776930.142029:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776930.142032:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776930.142033:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776930.142035:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776930.142038:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142045:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.142048:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142053:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.142055:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776930.142057:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776930.142059:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776930.142062:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776930.142063:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313317895 00000100:00000001:2.0:1775776930.142065:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313319158 00000100:00000001:2.0:1775776930.142067:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313317895 00000100:00000001:2.0:1775776930.142069:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313319158 00000100:00000001:2.0:1775776930.142071:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776930.142073:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.142075:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776930.142078:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776930/real 1775776930] req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776930.142084:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776930.142086:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776930.142088:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776930.142090:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776930.142091:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776930.142092:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776930.142095:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776930.142103:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142108:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776930.142110:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142111:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.142112:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776930.142115:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142122:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776930.142125:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142131:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776930.142133:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142134:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776930.142135:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142137:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776930.142138:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776930.142141:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776930.142144:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776930.142145:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776930.142149:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776930.142156:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099812864:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776930.142159:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142161:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.142163:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099812352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142169:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776930.142171:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776930.142174:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776930.142176:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776930.142178:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776930.142179:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776930.142181:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776930.142184:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115ac00. 02000000:00000001:2.0:1775776930.142187:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776930.142188:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776930.142190:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:2.0:1775776930.142193:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776930.142194:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142195:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.142198:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee300 x1862033099812480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142203:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776930.142204:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776930.142206:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776930.142208:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776930.142209:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776930.142210:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776930.142212:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776930.142213:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:2.0:1775776930.142215:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776930.142216:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776930.142217:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee300. 00000100:00000001:2.0:1775776930.142219:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776930.142220:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142222:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.142225:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded500 x1862033099812608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142230:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776930.142232:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776930.142233:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776930.142235:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776930.142236:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776930.142237:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776930.142238:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776930.142239:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:2.0:1775776930.142241:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776930.142242:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776930.142243:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded500. 00000100:00000001:2.0:1775776930.142245:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776930.142246:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142248:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.142250:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099812736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142256:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776930.142257:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776930.142258:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776930.142260:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776930.142261:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776930.142262:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776930.142264:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776930.142265:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:2.0:1775776930.142267:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776930.142267:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776930.142268:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:2.0:1775776930.142271:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776930.142272:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142273:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776930.142276:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec380 x1862033099812864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776930.142281:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776930.142282:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776930.142283:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776930.142286:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776930.142286:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776930.142287:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776930.142289:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776930.142291:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:2.0:1775776930.142293:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776930.142293:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776930.142294:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec380. 00000100:00000001:2.0:1775776930.142296:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776930.142298:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142300:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776930.142307:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776930.142321:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776930.142324:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776930.142325:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776930.844191:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776930.844198:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776930.844199:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776930.844204:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776930.844208:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776930.844212:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776930.844214:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776930.844217:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776930.844220:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907842:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776930.907848:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907854:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776930.907857:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776930.907859:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776930.907863:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907866:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776930.907868:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907870:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907872:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907875:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776930.907877:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776930.907878:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776930.907879:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907881:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776930.907882:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907884:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907885:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776930.907890:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776930.907891:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776930.907892:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907894:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776930.907895:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907896:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776930.907898:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776930.907976:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776930.907980:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776930.907982:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776931.035877:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776931.035883:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776931.035884:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776931.035888:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776931.035890:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776931.035893:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776931.035896:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776931.035898:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776931.035899:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776931.419943:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313319158, at 4313319168 00000100:00000010:0.0:1775776931.419959:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000040:1.0:1775776931.419959:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313319158, at 4313319168 00000100:00000010:1.0:1775776931.419967:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:3.0:1775776931.867959:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776931.867968:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776931.867969:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776931.867975:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776931.867978:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776931.867982:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776931.867984:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776931.867986:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776931.867989:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776931.932418:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776931.932425:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776931.932431:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776931.932432:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776931.932435:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776931.932437:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776931.932440:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776931.932441:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776931.932511:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776931.932514:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776931.932518:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776932.059900:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776932.059908:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776932.059910:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776932.059915:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776932.059919:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776932.059922:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776932.059925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776932.059927:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776932.059930:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776932.891879:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776932.891885:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776932.891887:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776932.891892:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776932.891896:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776932.891899:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776932.891902:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776932.891904:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776932.891906:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776932.955988:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776932.955995:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776932.956001:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776932.956002:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776932.956006:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776932.956007:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776932.956010:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776932.956011:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776932.958331:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776932.958336:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776932.958340:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776933.084614:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776933.084620:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776933.084621:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776933.084625:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776933.084628:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776933.084631:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776933.084632:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776933.084634:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776933.084636:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776933.915908:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776933.915915:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776933.915917:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776933.915923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776933.915927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776933.915930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776933.915933:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776933.915935:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776933.915938:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776933.982346:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776933.982352:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776933.982358:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776933.982359:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776933.982362:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776933.982363:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776933.982366:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776933.982367:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776933.982741:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776933.982745:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776933.982749:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776934.107906:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776934.107912:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776934.107914:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776934.107919:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776934.107923:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776934.107926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776934.107929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776934.107931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776934.107934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776934.939910:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776934.939919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776934.939921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776934.939927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776934.939931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776934.939935:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776934.939937:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776934.939939:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776934.939941:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776935.003915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776935.003922:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776935.003928:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776935.003929:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776935.003933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776935.003934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776935.003937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776935.003938:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776935.006315:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776935.006321:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776935.006326:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776935.132065:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776935.132071:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776935.132074:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776935.132080:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776935.132083:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776935.132088:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776935.132090:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776935.132093:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776935.132096:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776935.260355:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776935.260365:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776935.260368:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776935.260372:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776935.260377:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776935.260383:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18650 00000400:00000200:2.0:1775776935.260393:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776935.260400:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776935.260403:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776935.260406:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260410:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.260412:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260415:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776935.260418:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776935.260420:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776935.260423:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776935.260425:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776935.260428:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776935.260430:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776935.260433:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260435:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.260436:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260438:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776935.260441:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776935.260444:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260447:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776935.260450:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776935.260453:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776935.260456:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260458:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776935.260460:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776935.260469:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7dc00. 00000020:00000040:2.0:1775776935.260473:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776935.260476:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776935.260478:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260481:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776935.260483:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776935.260487:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776935.260494:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775776935.260500:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776935.260508:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776935.260519:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776935.260524:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776935.260529:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f1188f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776935.260555:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776935.260558:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776935.260561:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776935.260562:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776935.260564:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776935.260566:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776935.260569:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18650 00000400:00000200:2.0:1775776935.260572:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776935.260575:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776935.260577:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776935.260580:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260582:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.260584:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260586:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776935.260588:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776935.260589:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776935.260591:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776935.260592:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776935.260594:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776935.260596:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776935.260599:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260601:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.260603:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260605:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776935.260607:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776935.260609:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260611:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776935.260613:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776935.260617:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776935.260619:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260621:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776935.260625:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776935.260627:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260629:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776935.260632:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d880. 00000020:00000040:2.0:1775776935.260634:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776935.260636:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776935.260637:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260639:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776935.260640:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776935.260641:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776935.260644:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775776935.260647:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776935.260652:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776935.260660:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776935.260662:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776935.260666:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f88f35f] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776935.260673:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776935.260675:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776935.260679:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776935.260680:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776935.260682:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776935.260685:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776935.260687:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18650 00000400:00000200:2.0:1775776935.260690:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776935.260692:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776935.260694:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776935.260697:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260699:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.260701:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260703:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776935.260705:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776935.260706:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776935.260707:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776935.260709:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776935.260711:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776935.260712:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776935.260715:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260717:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.260718:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260721:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776935.260723:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776935.260725:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260728:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776935.260729:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776935.260731:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776935.260733:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260735:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776935.260737:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776935.260739:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260740:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776935.260743:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f480. 00000020:00000040:2.0:1775776935.260745:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776935.260747:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776935.260748:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260749:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776935.260750:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776935.260752:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776935.260755:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505c00. 00000100:00000001:2.0:1775776935.260758:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776935.260761:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776935.260768:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776935.260770:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776935.260773:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002c91b8b4] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776935.260808:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776935.260811:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776935.260814:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776935.260815:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776935.260817:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776935.260820:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776935.260822:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18650 00000400:00000200:2.0:1775776935.260826:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776935.260828:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776935.260831:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776935.260833:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260835:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.260837:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260839:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776935.260841:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776935.260843:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776935.260844:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776935.260846:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776935.260847:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776935.260849:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776935.260852:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260854:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.260856:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260858:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776935.260860:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776935.260862:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776935.260864:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776935.260866:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776935.260868:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776935.260870:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260872:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776935.260874:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776935.260876:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260877:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776935.260881:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f100. 00000020:00000040:2.0:1775776935.260883:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776935.260885:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776935.260886:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.260888:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776935.260889:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776935.260891:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776935.260894:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506400. 00000100:00000001:2.0:1775776935.260897:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776935.260902:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776935.260909:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776935.260911:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776935.260914:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048605922] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776935.260921:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776935.260924:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776935.260927:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776935.260928:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776935.261136:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776935.261140:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776935.261143:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776935.261148:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261156:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776935.261158:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.261162:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776935.261164:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776935.261166:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776935.261170:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099812992:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776935.261173:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776935.261177:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776935.261180:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.261182:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776935.261186:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776935.261188:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776935.261194:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776935.261197:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776935.261203:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776935.261210:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099812992, portal 25 00000100:00000001:0.0:1775776935.261212:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776935.261214:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288717824 : -114841420833792 : ffff978d65e7dc00) 00000100:00000040:0.0:1775776935.261246:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261255:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776935.261258:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776935.261261:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776935.261264:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099812992, offset 0 00000400:00000200:0.0:1775776935.261271:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776935.261278:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.261283:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776935.261288:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776935.261292:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776935.261295:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776935.261297:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776935.261301:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4582:4582:256:4294967295] 192.168.201.139@tcp LPNI seq info [4582:4582:8:4294967295] 00000400:00000200:0.0:1775776935.261306:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.261315:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776935.261321:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.261326:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776935.261332:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776935.261336:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776935.261342:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776935.261345:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776935.261362:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.261365:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776935.261371:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776935.261376:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776935.261381:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776935.261383:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776935.261388:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776935.261392:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261401:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.261404:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261411:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.261413:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776935.261419:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776935.261423:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776935.261425:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313324278 00000100:00000010:0.0:1775776935.261430:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776935.261435:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776935.261442:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313324278 00000100:00000010:0.0:1775776935.261445:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237360. 00000100:00000001:0.0:1775776935.261448:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776935.261451:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776935.261453:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.261455:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776935.261459:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776935/real 1775776935] req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776935.261468:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776935.261470:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776935.261472:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776935.261474:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776935.261476:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776935.261478:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776935.261481:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776935.261490:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261497:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776935.261498:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.261500:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.261502:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776935.261506:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261512:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776935.261516:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261523:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776935.261526:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.261528:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.261529:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.261531:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776935.261533:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776935.261536:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776935.261540:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776935.261542:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776935.261546:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776935.261554:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099812992:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776935.261558:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776935.261561:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261568:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776935.261569:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.261571:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776935.261573:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776935.261575:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776935.261577:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813120:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776935.261581:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776935.261582:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776935.261584:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.261586:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776935.261588:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776935.261590:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776935.261594:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776935.261596:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776935.261599:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776935.261602:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099813120, portal 10 00000100:00000001:0.0:1775776935.261604:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776935.261605:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716928 : -114841420834688 : ffff978d65e7d880) 00000100:00000040:0.0:1775776935.261611:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261617:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776935.261620:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776935.261622:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776935.261624:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099813120, offset 0 00000400:00000200:0.0:1775776935.261628:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776935.261631:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.261634:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776935.261637:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776935.261639:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776935.261642:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776935.261644:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776935.261647:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4583:4583:256:4294967295] 192.168.201.139@tcp LPNI seq info [4583:4583:8:4294967295] 00000400:00000200:0.0:1775776935.261651:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.261657:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776935.261661:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.261664:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776935.261668:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776935.261671:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776935.261675:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776935.261677:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776935.261681:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.261683:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776935.261686:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776935.261689:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776935.261691:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776935.261693:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776935.261695:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776935.261698:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261712:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.261714:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261721:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.261722:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776935.261724:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776935.261728:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776935.261731:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776935.261733:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313323013 00000100:00000001:0.0:1775776935.261735:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313324278 00000100:00000001:0.0:1775776935.261737:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313323013 00000100:00000001:0.0:1775776935.261739:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313324278 00000100:00000001:0.0:1775776935.261741:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776935.261743:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.261745:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776935.261748:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776935/real 1775776935] req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776935.261755:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776935.261756:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776935.261757:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776935.261759:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776935.261761:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776935.261763:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776935.261765:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776935.261773:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261823:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776935.261824:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.261826:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.261827:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776935.261831:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261838:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776935.261842:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261849:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776935.261851:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.261853:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.261854:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.261855:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776935.261857:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776935.261859:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776935.261862:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776935.261864:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776935.261868:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776935.261876:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813120:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776935.261879:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776935.261883:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.261888:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776935.261890:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.261892:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776935.261893:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776935.261895:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776935.261898:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813248:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776935.261901:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776935.261903:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776935.261904:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.261906:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776935.261909:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775776935.261911:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776935.261914:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776935.261917:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776935.261920:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776935.261922:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099813248, portal 10 00000100:00000001:0.0:1775776935.261924:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776935.261925:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724096 : -114841420827520 : ffff978d65e7f480) 00000100:00080000:2.0:1775776935.262194:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776935.262199:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776935.262202:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18650 00000400:00000200:2.0:1775776935.262206:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776935.262209:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776935.262212:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776935.262214:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.262217:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.262228:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.262230:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776935.262233:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776935.262234:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776935.262237:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776935.262239:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776935.262241:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776935.262243:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776935.262246:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.262248:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776935.262250:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776935.262252:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776935.262254:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776935.262257:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776935.262258:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776935.262260:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776935.262264:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776935.262266:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.262268:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776935.262271:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776935.262273:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.262275:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776935.262279:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d500. 00000020:00000040:2.0:1775776935.262282:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776935.262284:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776935.262285:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776935.262288:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776935.262289:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776935.262292:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776935.262296:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505400. 00000100:00000001:2.0:1775776935.262301:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776935.262305:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776935.262314:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776935.262317:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776935.262320:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000855db814] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776935.262330:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776935.262335:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18660) 00000100:00000040:0.0:1775776935.262390:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262397:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776935.262400:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776935.262403:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776935.262406:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099813248, offset 0 00000400:00000200:0.0:1775776935.262411:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776935.262415:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.262419:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776935.262422:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776935.262424:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776935.262427:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776935.262429:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776935.262432:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4584:4584:256:4294967295] 192.168.201.139@tcp LPNI seq info [4584:4584:8:4294967295] 00000400:00000200:0.0:1775776935.262436:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.262442:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776935.262447:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.262450:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776935.262454:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776935.262457:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776935.262461:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776935.262464:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776935.262468:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.262470:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776935.262473:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776935.262477:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776935.262479:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776935.262481:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776935.262484:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776935.262487:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262495:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.262498:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262504:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.262506:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776935.262509:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776935.262512:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776935.262515:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776935.262517:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313323015 00000100:00000001:0.0:1775776935.262519:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313324278 00000100:00000001:0.0:1775776935.262521:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313323015 00000100:00000001:0.0:1775776935.262523:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313324278 00000100:00000001:0.0:1775776935.262525:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776935.262527:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.262528:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776935.262532:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776935/real 1775776935] req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776935.262539:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776935.262541:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776935.262543:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776935.262545:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776935.262546:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776935.262548:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776935.262551:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776935.262559:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262565:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776935.262566:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.262568:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.262569:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776935.262572:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262578:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776935.262582:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262588:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776935.262591:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.262592:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.262593:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.262595:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776935.262596:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776935.262599:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776935.262602:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776935.262604:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776935.262607:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776935.262620:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813248:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776935.262624:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776935.262627:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262633:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776935.262634:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.262636:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776935.262638:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776935.262640:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776935.262643:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813376:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776935.262646:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776935.262648:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776935.262650:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.262651:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776935.262655:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776935.262657:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776935.262660:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776935.262662:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776935.262666:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776935.262668:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099813376, portal 4 00000100:00000001:0.0:1775776935.262670:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776935.262671:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288723200 : -114841420828416 : ffff978d65e7f100) 00000100:00000040:0.0:1775776935.262676:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262683:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776935.262685:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776935.262687:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776935.262689:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099813376, offset 0 00000400:00000200:0.0:1775776935.262692:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776935.262696:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.262699:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776935.262701:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776935.262704:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776935.262706:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776935.262708:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776935.262711:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4585:4585:256:4294967295] 192.168.201.139@tcp LPNI seq info [4585:4585:8:4294967295] 00000400:00000200:0.0:1775776935.262715:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.262721:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776935.262726:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.262729:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776935.262733:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776935.262735:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776935.262739:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776935.262741:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776935.262745:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.262747:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776935.262750:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776935.262753:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776935.262756:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776935.262757:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776935.262759:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776935.262762:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262770:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.262772:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262795:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.262797:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776935.262800:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776935.262803:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776935.262806:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776935.262808:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313323025 00000100:00000001:0.0:1775776935.262810:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313324278 00000100:00000001:0.0:1775776935.262812:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313323025 00000100:00000001:0.0:1775776935.262814:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313324278 00000100:00000001:0.0:1775776935.262816:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776935.262818:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.262820:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776935.262824:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776935/real 1775776935] req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776935.262831:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776935.262832:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776935.262834:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776935.262836:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776935.262837:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776935.262839:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776935.262842:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776935.262850:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262856:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776935.262858:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.262859:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.262860:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776935.262864:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262870:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776935.262874:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262880:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776935.262883:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.262884:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.262885:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.262887:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776935.262889:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776935.262891:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776935.262894:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776935.262896:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776935.262899:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776935.262907:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813376:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776935.262910:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.262912:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.262915:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7dc00 x1862033099812992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.262922:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776935.262924:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776935.262926:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776935.263025:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776935.263028:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776935.263029:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776935.263032:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776935.263034:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:0.0:1775776935.263038:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776935.263039:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776935.263041:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7dc00. 00000100:00000001:0.0:1775776935.263044:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776935.263045:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263047:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.263051:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d880 x1862033099813120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263063:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776935.263065:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776935.263066:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776935.263068:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776935.263069:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776935.263070:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776935.263072:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776935.263073:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:0.0:1775776935.263075:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776935.263076:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776935.263077:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d880. 00000100:00000001:0.0:1775776935.263079:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776935.263080:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263082:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.263084:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f480 x1862033099813248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263090:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776935.263091:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776935.263093:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775776935.263095:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776935.263096:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776935.263097:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776935.263099:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776935.263100:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505c00. 02000000:00000001:0.0:1775776935.263102:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776935.263102:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776935.263104:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f480. 00000100:00000001:0.0:1775776935.263105:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776935.263106:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263108:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.263110:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f100 x1862033099813376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263116:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776935.263117:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776935.263119:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776935.263121:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776935.263122:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776935.263123:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776935.263125:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776935.263126:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506400. 02000000:00000001:0.0:1775776935.263128:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776935.263129:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776935.263130:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f100. 00000100:00000001:0.0:1775776935.263132:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776935.263133:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263135:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263141:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776935.263143:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.263145:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776935.263146:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776935.263147:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776935.263150:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263157:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776935.263158:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.263160:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776935.263161:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776935.263163:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776935.263166:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813504:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776935.263169:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776935.263171:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776935.263172:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776935.263174:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776935.263177:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776935.263179:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776935.263182:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776935.263184:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776935.263188:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776935.263191:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099813504, portal 4 00000100:00000001:0.0:1775776935.263193:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776935.263194:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716032 : -114841420835584 : ffff978d65e7d500) 00000100:00000040:0.0:1775776935.263199:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263206:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776935.263208:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776935.263210:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776935.263212:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099813504, offset 0 00000400:00000200:0.0:1775776935.263215:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776935.263229:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.263233:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776935.263236:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776935.263238:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776935.263241:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776935.263243:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776935.263246:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4586:4586:256:4294967295] 192.168.201.139@tcp LPNI seq info [4586:4586:8:4294967295] 00000400:00000200:0.0:1775776935.263250:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776935.263256:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776935.263260:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.263263:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776935.263267:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776935.263269:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776935.263273:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776935.263276:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776935.263279:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776935.263281:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776935.263285:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776935.263288:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776935.263291:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776935.263292:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776935.263295:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776935.263298:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263312:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.263314:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263321:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.263322:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776935.263325:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776935.263327:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776935.263330:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776935.263332:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313323016 00000100:00000001:0.0:1775776935.263334:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313324278 00000100:00000001:0.0:1775776935.263336:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313323016 00000100:00000001:0.0:1775776935.263338:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313324278 00000100:00000001:0.0:1775776935.263340:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776935.263342:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.263344:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776935.263347:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776935/real 1775776935] req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776935.263355:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776935.263356:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776935.263358:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776935.263360:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776935.263361:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776935.263363:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776935.263366:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776935.263374:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263381:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776935.263382:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263384:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.263385:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776935.263389:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263395:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776935.263399:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263405:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776935.263407:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263409:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776935.263410:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263412:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776935.263413:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776935.263416:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776935.263419:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776935.263421:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776935.263424:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776935.263432:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813504:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776935.263436:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263437:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776935.263440:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d500 x1862033099813504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776935.263446:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776935.263447:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776935.263449:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776935.263450:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776935.263451:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776935.263453:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776935.263455:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776935.263456:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505400. 02000000:00000001:0.0:1775776935.263458:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776935.263459:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776935.263460:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d500. 00000100:00000001:0.0:1775776935.263463:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776935.263464:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263466:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776935.263469:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776935.263470:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776935.263471:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776935.263473:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776935.964003:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776935.964013:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776935.964015:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776935.964022:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776935.964027:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776935.964032:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776935.964035:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776935.964038:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776935.964041:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776936.028826:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776936.028834:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776936.028842:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776936.028843:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776936.028848:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776936.028849:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776936.028854:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776936.028855:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776936.028953:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776936.028959:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776936.028965:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776936.155892:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776936.155898:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776936.155900:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776936.155905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776936.155909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776936.155913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776936.155916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776936.155918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776936.155921:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776936.539888:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313324278, at 4313324288 00000100:00000010:1.0:1775776936.539903:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237360. 00000100:00000040:0.0:1775776936.539904:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313324278, at 4313324288 00000100:00000010:0.0:1775776936.539908:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775776936.987927:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776936.987935:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776936.987938:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776936.987944:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776936.987948:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776936.987952:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776936.987955:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776936.987957:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776936.987960:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776937.051874:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776937.051880:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776937.051886:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776937.051887:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776937.051891:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776937.051892:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776937.051895:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776937.051896:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776937.051989:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776937.051993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776937.051995:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776937.179938:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776937.179946:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776937.179947:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776937.179952:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776937.179956:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776937.179960:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776937.179962:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776937.179965:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776937.179967:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776938.012060:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776938.012067:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776938.012069:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776938.012075:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776938.012079:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776938.012083:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776938.012085:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776938.012088:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776938.012091:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776938.075908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776938.075913:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776938.075917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776938.075918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776938.075920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776938.075921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776938.075923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776938.075924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776938.076113:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776938.076117:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776938.076120:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776938.203982:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776938.203987:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776938.203988:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776938.203992:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776938.203994:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776938.203997:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776938.203999:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776938.204001:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776938.204003:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776939.035925:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776939.035931:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776939.035933:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776939.035939:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776939.035942:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776939.035946:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776939.035948:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776939.035950:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776939.035953:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776939.099857:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776939.099863:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776939.099866:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776939.099869:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776939.099873:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776939.099875:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776939.099877:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776939.099879:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776939.099883:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776939.099884:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776939.099888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776939.099889:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776939.099892:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776939.099893:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776939.100102:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776939.100106:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776939.100109:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776939.228884:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776939.228890:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776939.228892:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776939.228897:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776939.228900:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776939.228903:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776939.228906:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776939.228908:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776939.228911:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776940.060036:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776940.060043:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776940.060044:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776940.060049:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776940.060053:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776940.060057:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776940.060059:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776940.060062:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776940.060065:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776940.123906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776940.123911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776940.123917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776940.123918:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776940.123921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776940.123922:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776940.123925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776940.123926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776940.124019:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776940.124022:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776940.124025:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776940.251897:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776940.251904:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776940.251906:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776940.251910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776940.252115:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776940.252119:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776940.252122:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776940.252124:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776940.252127:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776940.382906:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776940.382916:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776940.382919:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776940.382922:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776940.382926:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776940.382933:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18655 00000400:00000200:0.0:1775776940.382942:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776940.382948:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776940.382951:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776940.382954:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.382957:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.382959:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.382962:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776940.382966:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776940.382968:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776940.382971:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776940.382974:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776940.382976:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776940.382979:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776940.382981:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.382983:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.382984:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.382986:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776940.382989:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776940.382992:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776940.382994:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776940.382997:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776940.383000:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776940.383002:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383005:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776940.383007:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776940.383014:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775776940.383018:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776940.383021:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776940.383023:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383025:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776940.383028:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776940.383032:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776940.383039:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775776940.383045:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776940.383053:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776940.383063:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776940.383067:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776940.383071:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776940.383100:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776940.383103:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776940.383106:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776940.383107:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776940.383109:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776940.383112:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776940.383114:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18655 00000400:00000200:0.0:1775776940.383117:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776940.383120:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776940.383123:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776940.383125:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383127:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.383156:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383159:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776940.383161:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776940.383162:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776940.383164:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776940.383166:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776940.383168:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776940.383170:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776940.383173:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383174:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.383176:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383178:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776940.383180:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776940.383183:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383185:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776940.383187:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776940.383192:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776940.383195:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383197:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776940.383201:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776940.383204:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383205:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776940.383209:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775776940.383211:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776940.383213:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776940.383214:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383216:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776940.383217:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776940.383219:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776940.383222:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775776940.383286:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776940.383293:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776940.383301:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776940.383303:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776940.383307:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776940.383314:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776940.383317:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776940.383320:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776940.383322:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776940.383323:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776940.383326:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776940.383328:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18655 00000400:00000200:0.0:1775776940.383331:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776940.383334:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776940.383336:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776940.383339:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383341:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.383343:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383345:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776940.383347:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776940.383348:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776940.383349:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776940.383351:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776940.383353:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776940.383355:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776940.383357:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383359:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.383361:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383363:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776940.383365:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776940.383367:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383369:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776940.383370:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776940.383373:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776940.383374:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383376:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776940.383378:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776940.383380:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383382:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776940.383385:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775776940.383387:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776940.383389:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776940.383390:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383391:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776940.383392:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776940.383394:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776940.383397:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776940.383400:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776940.383404:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776940.383411:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776940.383413:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776940.383416:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776940.383423:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776940.383425:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776940.383428:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776940.383429:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776940.383431:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776940.383433:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776940.383435:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18655 00000400:00000200:0.0:1775776940.383438:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776940.383441:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776940.383443:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776940.383445:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383447:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.383448:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383450:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776940.383452:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776940.383453:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776940.383454:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776940.383456:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776940.383458:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776940.383460:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776940.383462:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383464:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.383466:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383467:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776940.383469:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776940.383471:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383474:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776940.383475:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776940.383478:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776940.383479:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383481:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776940.383483:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776940.383485:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383486:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776940.383490:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775776940.383492:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776940.383495:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776940.383496:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383498:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776940.383499:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776940.383501:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776940.383504:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775776940.383507:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776940.383511:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776940.383518:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776940.383520:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776940.383523:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776940.383530:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776940.383532:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776940.383535:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776940.383536:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776940.383539:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776940.383541:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776940.383543:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18655 00000400:00000200:0.0:1775776940.383547:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776940.383550:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776940.383552:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776940.383554:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383556:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.383558:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383560:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776940.383562:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776940.383563:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776940.383565:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776940.383566:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776940.383569:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776940.383571:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776940.383573:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383575:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776940.383577:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383579:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776940.383581:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776940.383584:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776940.383586:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776940.383587:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776940.383589:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776940.383591:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383593:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776940.383594:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776940.383596:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383598:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776940.383602:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776940.383604:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776940.383607:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776940.383608:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776940.383610:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776940.383611:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776940.383612:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776940.383615:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775776940.383618:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776940.383622:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776940.383629:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776940.383631:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776940.383633:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776940.383641:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776940.383644:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18665) 00000100:00000001:2.0:1775776940.383914:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776940.383918:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776940.383920:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776940.383925:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.383934:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776940.383936:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.383938:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776940.383941:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776940.383943:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776940.383946:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099813632:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776940.383950:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776940.383954:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776940.383956:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.383958:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776940.383962:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776940.383964:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776940.383969:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776940.383973:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776940.383979:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776940.383985:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099813632, portal 25 00000100:00000001:2.0:1775776940.383988:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776940.383989:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:2.0:1775776940.383994:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.384001:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776940.384004:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776940.384006:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776940.384009:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099813632, offset 0 00000400:00000200:2.0:1775776940.384015:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776940.384021:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.384026:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776940.384030:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776940.384034:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776940.384037:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776940.384040:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776940.384044:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4587:4587:256:4294967295] 192.168.201.139@tcp LPNI seq info [4587:4587:8:4294967295] 00000400:00000200:2.0:1775776940.384049:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.384058:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776940.384064:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.384068:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776940.384075:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776940.384078:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776940.384084:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776940.384087:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775776940.384091:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.384094:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776940.384100:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776940.384104:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776940.384108:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776940.384110:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776940.384114:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776940.384118:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.384126:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.384142:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.384150:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.384151:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776940.384157:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776940.384160:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776940.384163:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313329401 00000100:00000010:2.0:1775776940.384167:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b5a0. 00000100:00000001:2.0:1775776940.384172:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776940.384177:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313329401 00000100:00000010:2.0:1775776940.384180:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775776940.384182:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776940.384185:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776940.384187:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.384189:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776940.384194:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776940/real 1775776940] req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776940.384203:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776940.384205:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776940.384206:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776940.384208:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776940.384210:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776940.384212:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776940.384215:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776940.384224:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.384866:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776940.384868:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.384870:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.384872:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776940.384877:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.384884:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776940.384888:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.384895:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776940.384898:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.384899:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.384900:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.384902:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776940.384904:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776940.384906:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776940.384910:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776940.384912:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776940.384916:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776940.384924:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099813632:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776940.384928:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776940.384932:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.384938:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776940.384939:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.384941:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776940.384943:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776940.384944:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776940.384947:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813760:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776940.384950:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776940.384952:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776940.384954:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.384956:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776940.384959:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776940.384961:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776940.384965:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776940.384967:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776940.384971:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776940.384974:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099813760, portal 10 00000100:00000001:2.0:1775776940.384976:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776940.384977:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:2.0:1775776940.384983:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.384990:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776940.384992:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776940.384994:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776940.384996:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099813760, offset 0 00000400:00000200:2.0:1775776940.384999:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776940.385003:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.385006:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776940.385009:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776940.385011:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776940.385013:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776940.385015:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776940.385018:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4588:4588:256:4294967295] 192.168.201.139@tcp LPNI seq info [4588:4588:8:4294967295] 00000400:00000200:2.0:1775776940.385022:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.385029:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776940.385034:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.385037:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776940.385040:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776940.385043:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776940.385047:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776940.385049:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775776940.385053:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.385054:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776940.385058:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776940.385061:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776940.385064:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776940.385066:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776940.385068:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776940.385071:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385078:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.385081:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385087:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.385089:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776940.385091:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776940.385095:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776940.385098:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776940.385099:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313328136 00000100:00000001:2.0:1775776940.385102:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313329401 00000100:00000001:2.0:1775776940.385104:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313328136 00000100:00000001:2.0:1775776940.385106:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313329401 00000100:00000001:2.0:1775776940.385108:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776940.385110:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.385112:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776940.385115:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776940/real 1775776940] req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776940.385122:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776940.385124:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776940.385125:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776940.385127:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776940.385128:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776940.385143:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776940.385147:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776940.385155:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385161:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776940.385162:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.385164:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.385165:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776940.385169:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385175:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776940.385178:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385185:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776940.385187:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.385189:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.385190:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.385191:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776940.385193:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776940.385195:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776940.385198:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776940.385200:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776940.385204:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776940.385212:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813760:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776940.385215:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776940.385218:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385296:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776940.385297:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.385299:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776940.385301:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776940.385303:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776940.385306:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813888:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776940.385309:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776940.385311:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776940.385312:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.385314:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776940.385317:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776940.385319:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776940.385322:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776940.385325:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776940.385327:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776940.385330:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099813888, portal 10 00000100:00000001:2.0:1775776940.385332:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776940.385333:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:2.0:1775776940.385338:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385344:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776940.385346:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776940.385348:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776940.385350:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099813888, offset 0 00000400:00000200:2.0:1775776940.385354:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776940.385357:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.385361:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776940.385363:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776940.385366:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776940.385368:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776940.385370:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776940.385374:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4589:4589:256:4294967295] 192.168.201.139@tcp LPNI seq info [4589:4589:8:4294967295] 00000400:00000200:2.0:1775776940.385377:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.385384:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776940.385388:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.385391:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776940.385394:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776940.385397:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776940.385401:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776940.385403:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775776940.385406:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.385408:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776940.385411:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776940.385414:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776940.385416:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776940.385418:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776940.385420:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776940.385422:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385430:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.385432:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385438:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.385440:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776940.385442:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776940.385445:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776940.385447:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776940.385449:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313328137 00000100:00000001:2.0:1775776940.385451:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313329401 00000100:00000001:2.0:1775776940.385453:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313328137 00000100:00000001:2.0:1775776940.385455:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313329401 00000100:00000001:2.0:1775776940.385457:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776940.385459:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.385461:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776940.385464:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776940/real 1775776940] req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776940.385471:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776940.385472:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776940.385474:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776940.385476:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776940.385477:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776940.385478:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776940.385481:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776940.385489:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385495:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776940.385496:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.385498:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.385499:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776940.385503:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385509:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776940.385513:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385519:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776940.385521:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.385523:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.385524:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.385526:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776940.385527:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776940.385529:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776940.385533:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776940.385535:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776940.385539:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776940.385546:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099813888:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776940.385549:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776940.385552:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385558:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776940.385560:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.385561:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776940.385563:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776940.385565:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776940.385567:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814016:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776940.385570:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776940.385572:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776940.385573:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.385575:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776940.385578:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775776940.385580:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776940.385583:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776940.385585:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776940.385588:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776940.385591:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099814016, portal 4 00000100:00000001:2.0:1775776940.385592:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776940.385594:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000100:00000040:2.0:1775776940.385599:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385605:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776940.385607:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776940.385609:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776940.385612:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099814016, offset 0 00000400:00000200:2.0:1775776940.385615:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776940.385619:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.385622:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776940.385625:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776940.385627:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776940.385629:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776940.385631:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776940.385635:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4590:4590:256:4294967295] 192.168.201.139@tcp LPNI seq info [4590:4590:8:4294967295] 00000400:00000200:2.0:1775776940.385638:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.385644:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776940.385649:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.385652:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776940.385655:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776940.385658:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776940.385662:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776940.385664:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775776940.385667:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.385669:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776940.385672:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776940.385675:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776940.385677:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776940.385679:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776940.385681:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776940.385684:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385691:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.385694:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385700:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.385702:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776940.385705:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776940.385707:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776940.385710:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776940.385712:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313328146 00000100:00000001:2.0:1775776940.385714:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313329401 00000100:00000001:2.0:1775776940.385716:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313328146 00000100:00000001:2.0:1775776940.385718:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313329401 00000100:00000001:2.0:1775776940.385720:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776940.385721:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.385723:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776940.385726:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776940/real 1775776940] req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776940.385733:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776940.385735:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776940.385736:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776940.385738:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776940.385739:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776940.385741:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776940.385743:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776940.385752:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.385757:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776940.385758:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.385760:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.385761:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776940.386933:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.386941:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776940.386944:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.386952:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776940.386954:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.386956:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.386958:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.386959:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776940.386961:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776940.386963:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776940.386966:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776940.386969:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776940.386972:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776940.386979:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814016:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776940.386983:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776940.386987:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.386993:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776940.386994:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.386996:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776940.387006:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776940.387009:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776940.387012:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814144:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776940.387015:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776940.387020:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776940.387022:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776940.387023:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776940.387027:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776940.387029:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776940.387033:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776940.387035:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776940.387039:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776940.387042:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099814144, portal 4 00000100:00000001:2.0:1775776940.387044:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776940.387045:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748096 : -114841967803520 : ffff978d454dc380) 00000100:00000040:2.0:1775776940.387051:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387057:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776940.387059:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776940.387061:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776940.387064:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099814144, offset 0 00000400:00000200:2.0:1775776940.387068:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776940.387072:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.387076:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776940.387079:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776940.387082:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776940.387084:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776940.387086:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776940.387090:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4591:4591:256:4294967295] 192.168.201.139@tcp LPNI seq info [4591:4591:8:4294967295] 00000400:00000200:2.0:1775776940.387093:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776940.387100:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776940.387105:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.387108:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776940.387112:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776940.387114:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1775776940.387119:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776940.387121:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1775776940.387125:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776940.387127:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776940.387141:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776940.387144:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776940.387147:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776940.387149:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776940.387152:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776940.387155:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387163:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.387165:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387172:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.387173:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776940.387177:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776940.387179:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776940.387182:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776940.387184:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313328140 00000100:00000001:2.0:1775776940.387187:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313329401 00000100:00000001:2.0:1775776940.387189:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313328140 00000100:00000001:2.0:1775776940.387191:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313329401 00000100:00000001:2.0:1775776940.387192:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776940.387194:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.387196:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776940.387199:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776940/real 1775776940] req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776940.387207:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776940.387208:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776940.387210:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776940.387212:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776940.387213:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776940.387215:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776940.387218:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776940.387404:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387419:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776940.387420:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387422:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.387423:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776940.387427:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387433:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776940.387437:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387444:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776940.387446:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387447:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776940.387448:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387450:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776940.387451:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776940.387454:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776940.387457:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776940.387459:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776940.387463:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776940.387469:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814144:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776940.387473:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387475:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.387478:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099813632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387485:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776940.387487:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776940.387490:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775776940.387493:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776940.387495:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776940.387496:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776940.387498:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776940.387500:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:2.0:1775776940.387504:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776940.387506:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776940.387508:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:2.0:1775776940.387511:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776940.387512:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387513:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.387516:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099813760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387522:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776940.387523:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776940.387525:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775776940.387527:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776940.387528:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776940.387529:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776940.387530:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776940.387531:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:2.0:1775776940.387533:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776940.387535:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776940.387536:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:2.0:1775776940.387538:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776940.387539:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387541:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.387544:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099813888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387550:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776940.387551:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776940.387553:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776940.387555:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776940.387556:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776940.387557:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776940.387559:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776940.387560:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:2.0:1775776940.387562:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776940.387563:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776940.387564:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000100:00000001:2.0:1775776940.387566:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776940.387567:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387569:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.387572:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099814016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387577:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776940.387578:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776940.387579:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775776940.387581:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776940.387582:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776940.387583:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776940.387585:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776940.387586:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f800. 02000000:00000001:2.0:1775776940.387588:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776940.387589:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776940.387590:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:2.0:1775776940.387591:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776940.387593:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387594:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776940.387597:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc380 x1862033099814144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776940.387603:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776940.387604:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776940.387605:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776940.387607:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776940.387608:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776940.387609:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776940.387611:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776940.387612:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e400. 02000000:00000001:2.0:1775776940.387614:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776940.387616:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776940.387617:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc380. 00000100:00000001:2.0:1775776940.387619:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776940.387620:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387622:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776940.387629:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776940.387630:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776940.387632:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776940.387634:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776941.085081:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776941.085088:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776941.085090:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776941.085095:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776941.085098:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776941.085101:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776941.085104:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776941.085107:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776941.085109:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776941.147910:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776941.147919:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776941.147921:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776941.147925:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147928:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776941.147930:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147932:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147934:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776941.147939:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776941.147940:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776941.147941:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147943:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776941.147943:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147945:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147946:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776941.147951:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776941.147952:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776941.147953:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147955:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776941.147956:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147957:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776941.147959:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776941.148033:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776941.148036:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776941.148039:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776941.275885:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776941.275892:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776941.275894:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776941.275900:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776941.275904:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776941.275908:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776941.275911:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776941.275913:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776941.275917:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776941.659885:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313329401, at 4313329408 00000100:00000010:1.0:1775776941.659901:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000040:0.0:1775776941.659966:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313329401, at 4313329408 00000100:00000010:0.0:1775776941.659972:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000001:3.0:1775776942.108225:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776942.108231:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776942.108233:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776942.108238:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776942.108242:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776942.108245:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776942.108248:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776942.108251:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776942.108253:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776942.173240:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776942.173245:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776942.173250:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776942.173251:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776942.173254:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776942.173255:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776942.173258:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776942.173259:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776942.173311:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776942.173314:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776942.173316:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776942.299859:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776942.299866:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776942.299868:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776942.299873:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776942.299876:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776942.299880:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776942.299882:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776942.299884:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776942.299887:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776943.131892:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776943.131898:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776943.131899:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776943.131903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776943.131905:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776943.131912:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776943.131914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776943.131916:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776943.131918:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776943.195905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776943.195911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776943.195916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776943.195917:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776943.195920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776943.195921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776943.195924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776943.195925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776943.195979:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776943.195983:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776943.195986:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776943.323964:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776943.323970:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776943.323972:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776943.323978:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776943.323981:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776943.323985:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776943.323987:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776943.323989:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776943.323991:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776944.155894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776944.155900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776944.155901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776944.155905:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776944.155908:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776944.155912:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776944.155914:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776944.155918:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776944.155920:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776944.219942:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776944.219947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776944.219952:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776944.219953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776944.219956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776944.219957:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776944.219961:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776944.219962:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776944.220032:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776944.220035:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776944.220038:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776944.347935:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776944.347942:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776944.347947:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776944.347953:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776944.347957:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776944.347961:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776944.347964:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776944.347967:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776944.347970:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776945.179895:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776945.179902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776945.179904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776945.179908:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776945.179912:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776945.179916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776945.179918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776945.179920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776945.179922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776945.243913:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776945.243919:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776945.243924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776945.243925:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776945.243928:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776945.243929:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776945.243943:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776945.243944:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776945.244140:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776945.244144:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776945.244147:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776945.371909:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776945.371915:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776945.371916:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776945.371920:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776945.371924:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776945.371928:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776945.371930:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776945.371932:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776945.371935:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776945.499964:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776945.499974:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776945.499978:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776945.499982:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776945.499986:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776945.500014:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18660 00000400:00000200:1.0:1775776945.500023:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776945.500030:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776945.500033:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776945.500036:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500051:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.500054:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500057:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776945.500061:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776945.500063:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776945.500065:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776945.500068:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776945.500071:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776945.500073:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776945.500076:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500077:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.500080:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500082:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776945.500084:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776945.500088:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500090:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776945.500093:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776945.500096:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776945.500099:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.500101:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776945.500104:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776945.500114:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775776945.500117:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776945.500120:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776945.500122:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.500125:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776945.500127:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776945.500131:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776945.500139:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775776945.500145:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776945.500160:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776945.500171:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776945.500175:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776945.500180:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500248:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776945.500252:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776945.500255:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776945.500261:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500268:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776945.500270:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.500274:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776945.500276:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776945.500278:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776945.500281:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099814272:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776945.500285:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776945.500289:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776945.500291:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.500294:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776945.500298:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776945.500300:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776945.500304:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776945.500309:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776945.500316:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776945.500323:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099814272, portal 25 00000100:00000001:2.0:1775776945.500326:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776945.500327:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287607296 : -114841421944320 : ffff978d65d6ea00) 00000100:00000040:2.0:1775776945.500333:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500345:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776945.500347:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776945.500350:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776945.500353:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099814272, offset 0 00000400:00000200:2.0:1775776945.500357:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776945.500363:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.500368:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776945.500372:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776945.500375:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776945.500379:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776945.500381:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776945.500384:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4592:4592:256:4294967295] 192.168.201.139@tcp LPNI seq info [4592:4592:8:4294967295] 00000400:00000200:2.0:1775776945.500387:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.500395:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776945.500399:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.500403:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776945.500410:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776945.500414:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776945.500419:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776945.500422:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775776945.500426:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.500428:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776945.500434:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776945.500439:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776945.500445:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776945.500446:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776945.500451:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776945.500455:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776945.500480:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776945.500484:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776945.500487:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776945.500488:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776945.500490:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776945.500493:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776945.500496:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18660 00000400:00000200:1.0:1775776945.500500:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776945.500503:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776945.500505:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776945.500507:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500510:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.500512:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500514:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776945.500516:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776945.500517:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776945.500518:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776945.500520:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776945.500521:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776945.500523:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776945.500526:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500527:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.500529:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500531:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776945.500533:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776945.500535:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776945.500537:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776945.500539:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776945.500544:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776945.500546:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.500548:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776945.500553:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776945.500556:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.500557:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776945.500562:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775776945.500564:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776945.500567:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776945.500569:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.500570:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776945.500572:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776945.500574:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776945.500577:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775776945.500581:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776945.500585:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776945.500592:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776945.500594:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776945.500598:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500687:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.500692:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500701:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.500702:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776945.500712:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776945.500716:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776945.500719:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313334517 00000100:00000010:2.0:1775776945.500724:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bba0. 00000100:00000001:2.0:1775776945.500730:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776945.500737:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313334517 00000100:00000010:2.0:1775776945.500739:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b1e0. 00000100:00000001:2.0:1775776945.500742:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776945.500744:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776945.500747:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.500749:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776945.500753:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776945/real 1775776945] req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776945.500768:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776945.500770:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776945.500772:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776945.500774:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776945.500776:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776945.500827:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776945.500831:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776945.500839:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500845:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776945.500847:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.500849:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.500851:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776945.500854:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500861:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776945.500864:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500870:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776945.500873:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.500874:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.500875:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.500877:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776945.500878:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776945.500881:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776945.500884:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776945.500885:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776945.500889:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776945.500896:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099814272:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776945.500900:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.500902:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.500904:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ea00 x1862033099814272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500910:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776945.500912:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776945.500914:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776945.500916:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776945.500918:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776945.500919:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776945.500920:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776945.500922:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3bc00. 02000000:00000001:2.0:1775776945.500925:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776945.500926:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776945.500928:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ea00. 00000100:00000001:2.0:1775776945.500930:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776945.500931:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.500933:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.500939:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776945.500941:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.500942:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776945.500943:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776945.500944:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776945.500947:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500952:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776945.500953:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.500955:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776945.500956:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776945.500958:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776945.500960:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814400:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776945.500962:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776945.500964:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776945.500965:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.500967:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776945.500969:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776945.500971:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776945.500974:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776945.500976:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776945.500978:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776945.500981:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099814400, portal 10 00000100:00000001:2.0:1775776945.500983:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776945.500984:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287606400 : -114841421945216 : ffff978d65d6e680) 00000100:00000040:2.0:1775776945.500988:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.500993:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776945.500995:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776945.500997:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776945.500999:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099814400, offset 0 00000400:00000200:2.0:1775776945.501002:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776945.501005:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.501008:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776945.501010:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776945.501012:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776945.501015:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776945.501016:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776945.501019:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4593:4593:256:4294967295] 192.168.201.139@tcp LPNI seq info [4593:4593:8:4294967295] 00000400:00000200:2.0:1775776945.501023:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.501029:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776945.501034:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.501038:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776945.501058:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776945.501061:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776945.501065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776945.501068:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775776945.501072:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.501074:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776945.501077:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776945.501080:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776945.501083:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776945.501085:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776945.501087:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776945.501090:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.501098:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.501101:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.501107:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.501109:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776945.501111:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776945.501115:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776945.501118:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776945.501120:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313333253 00000100:00000001:2.0:1775776945.501122:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313334517 00000100:00000001:2.0:1775776945.501124:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313333253 00000100:00000001:2.0:1775776945.501125:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313334517 00000100:00000001:2.0:1775776945.501127:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776945.501129:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.501131:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776945.501134:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776945/real 1775776945] req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776945.501142:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776945.501143:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776945.501145:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776945.501147:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776945.501149:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776945.501151:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776945.501154:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776945.501162:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.501168:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776945.501170:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.501171:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.501172:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776945.501176:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.501182:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776945.501186:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.501192:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776945.501194:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.501196:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.501197:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.501199:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776945.501201:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776945.501203:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776945.501206:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776945.501208:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776945.501211:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776945.501218:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814400:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776945.501222:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.501224:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.501226:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6e680 x1862033099814400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.501232:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776945.501233:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776945.501235:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776945.501237:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776945.501238:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776945.501239:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776945.501255:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776945.501256:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39000. 02000000:00000001:2.0:1775776945.501258:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776945.501259:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776945.501261:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6e680. 00000100:00000001:2.0:1775776945.501262:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776945.501263:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.501265:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.501268:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776945.501269:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.501271:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776945.501272:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501462:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776945.501466:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776945.501470:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776945.501472:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776945.501474:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776945.501477:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776945.501480:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18660 00000100:00000001:2.0:1775776945.501483:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1775776945.501485:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776945.501488:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:2.0:1775776945.501490:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776945.501491:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776945.501496:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501498:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.501501:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501503:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776945.501506:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776945.501507:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776945.501512:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776945.501514:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776945.501516:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776945.501519:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776945.501524:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501526:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.501527:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501529:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776945.501531:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776945.501533:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501535:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776945.501536:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776945.501540:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776945.501542:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501543:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776945.501546:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776945.501548:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501550:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776945.501553:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775776945.501555:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776945.501558:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776945.501559:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501561:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776945.501562:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776945.501564:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776945.501568:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775776945.501571:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776945.501576:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776945.501584:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776945.501587:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776945.501590:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776945.501612:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776945.501616:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776945.501619:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776945.501621:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776945.501622:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776945.501625:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776945.501627:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18660 00000400:00000200:1.0:1775776945.501631:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776945.501634:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776945.501637:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776945.501639:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501641:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.501643:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501645:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776945.501647:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776945.501648:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776945.501650:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776945.501652:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776945.501654:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776945.501656:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776945.501658:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501660:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.501662:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501663:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776945.501667:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776945.501670:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501672:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776945.501673:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776945.501676:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776945.501678:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501680:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776945.501683:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776945.501685:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501687:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776945.501690:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775776945.501692:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776945.501694:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776945.501696:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501697:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776945.501699:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776945.501700:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776945.501703:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776945.501707:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776945.501711:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776945.501718:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776945.501720:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776945.501723:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776945.501730:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776945.501732:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776945.501735:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776945.501736:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776945.501737:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776945.501740:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776945.501742:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18660 00000400:00000200:1.0:1775776945.501745:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776945.501748:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776945.501750:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776945.501753:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501755:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.501757:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501759:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776945.501761:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776945.501762:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776945.501764:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776945.501766:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776945.501767:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776945.501769:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776945.501772:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501774:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776945.501775:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501777:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776945.501797:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776945.501799:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776945.501802:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776945.501803:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776945.501806:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776945.501808:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501810:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776945.501812:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776945.501814:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501816:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776945.501820:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775776945.501822:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776945.501824:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776945.501825:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776945.501826:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776945.501828:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776945.501829:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776945.501832:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776945.501835:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776945.501839:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776945.501851:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776945.501853:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776945.501857:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776945.501863:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776945.501867:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18670) 00000100:00000001:2.0:1775776945.501922:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776945.501924:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776945.501926:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776945.501930:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.501937:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776945.501939:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.501942:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776945.501943:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776945.501945:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776945.501948:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814528:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776945.501951:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776945.501954:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776945.501955:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.501956:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776945.501960:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776945.501962:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776945.501966:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776945.501969:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776945.501973:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776945.501976:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099814528, portal 10 00000100:00000001:2.0:1775776945.501978:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776945.501979:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601920 : -114841421949696 : ffff978d65d6d500) 00000100:00000040:2.0:1775776945.501985:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.501991:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776945.501994:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776945.501996:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776945.501998:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099814528, offset 0 00000400:00000200:2.0:1775776945.502003:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776945.502007:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.502011:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776945.502015:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776945.502017:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776945.502020:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776945.502023:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776945.502026:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4594:4594:256:4294967295] 192.168.201.139@tcp LPNI seq info [4594:4594:8:4294967295] 00000400:00000200:2.0:1775776945.502030:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.502036:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776945.502056:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.502060:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776945.502065:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776945.502068:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776945.502072:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776945.502074:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775776945.502078:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.502080:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776945.502084:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776945.502087:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776945.502090:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776945.502092:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776945.502095:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776945.502098:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502105:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.502108:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502114:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.502116:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776945.502119:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776945.502122:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776945.502125:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776945.502127:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313333255 00000100:00000001:2.0:1775776945.502129:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313334517 00000100:00000001:2.0:1775776945.502131:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313333255 00000100:00000001:2.0:1775776945.502132:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313334517 00000100:00000001:2.0:1775776945.502134:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776945.502136:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.502138:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776945.502142:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776945/real 1775776945] req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776945.502149:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776945.502150:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776945.502153:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776945.502155:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776945.502156:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776945.502159:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776945.502162:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776945.502169:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502176:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776945.502177:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502179:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.502181:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776945.502185:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502191:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776945.502194:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502200:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776945.502202:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502203:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.502204:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502206:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776945.502207:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776945.502210:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776945.502213:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776945.502215:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776945.502218:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776945.502226:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814528:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776945.502230:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776945.502232:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502237:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776945.502238:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.502240:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776945.502242:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776945.502243:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776945.502246:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814656:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776945.502248:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776945.502250:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776945.502251:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.502253:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776945.502256:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775776945.502258:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776945.502261:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776945.502263:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776945.502266:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776945.502268:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099814656, portal 4 00000100:00000001:2.0:1775776945.502270:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776945.502271:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287604608 : -114841421947008 : ffff978d65d6df80) 00000100:00000040:2.0:1775776945.502275:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502280:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776945.502282:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776945.502283:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776945.502285:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099814656, offset 0 00000400:00000200:2.0:1775776945.502289:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776945.502292:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.502295:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776945.502298:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776945.502300:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776945.502302:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776945.502304:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776945.502306:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4595:4595:256:4294967295] 192.168.201.139@tcp LPNI seq info [4595:4595:8:4294967295] 00000400:00000200:2.0:1775776945.502310:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.502315:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776945.502319:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.502322:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776945.502325:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776945.502327:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776945.502331:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776945.502333:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775776945.502337:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.502338:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776945.502341:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776945.502345:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776945.502347:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776945.502349:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776945.502351:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776945.502354:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502361:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.502364:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502370:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.502371:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776945.502374:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776945.502377:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776945.502380:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776945.502381:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313333264 00000100:00000001:2.0:1775776945.502384:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313334517 00000100:00000001:2.0:1775776945.502386:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313333264 00000100:00000001:2.0:1775776945.502388:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313334517 00000100:00000001:2.0:1775776945.502390:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776945.502392:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.502393:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776945.502397:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776945/real 1775776945] req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776945.502404:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776945.502405:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776945.502407:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776945.502409:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776945.502410:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776945.502412:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776945.502415:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776945.502422:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502429:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776945.502430:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502432:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.502433:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776945.502436:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502442:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776945.502446:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502452:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776945.502454:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502456:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.502457:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502459:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776945.502461:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776945.502463:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776945.502466:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776945.502468:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776945.502472:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776945.502480:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814656:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776945.502483:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776945.502486:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502492:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776945.502494:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.502495:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776945.502497:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776945.502499:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776945.502502:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814784:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776945.502505:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776945.502507:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776945.502508:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776945.502510:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776945.502513:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776945.502515:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776945.502518:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776945.502520:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776945.502523:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776945.502526:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099814784, portal 4 00000100:00000001:2.0:1775776945.502527:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776945.502528:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287599232 : -114841421952384 : ffff978d65d6ca80) 00000100:00000040:2.0:1775776945.502533:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502540:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776945.502542:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776945.502544:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776945.502547:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099814784, offset 0 00000400:00000200:2.0:1775776945.502550:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776945.502554:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.502557:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776945.502560:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776945.502562:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776945.502564:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776945.502567:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776945.502569:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4596:4596:256:4294967295] 192.168.201.139@tcp LPNI seq info [4596:4596:8:4294967295] 00000400:00000200:2.0:1775776945.502573:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776945.502579:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776945.502583:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.502586:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776945.502589:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776945.502592:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1775776945.502595:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776945.502597:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1775776945.502601:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776945.502603:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776945.502605:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776945.502609:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776945.502611:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776945.502613:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776945.502615:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776945.502618:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502625:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.502627:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502639:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.502640:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776945.502643:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776945.502645:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776945.502648:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776945.502650:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313333255 00000100:00000001:2.0:1775776945.502652:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313334517 00000100:00000001:2.0:1775776945.502654:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313333255 00000100:00000001:2.0:1775776945.502656:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313334517 00000100:00000001:2.0:1775776945.502658:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776945.502659:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.502661:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776945.502664:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776945/real 1775776945] req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776945.502672:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776945.502673:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776945.502674:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776945.502677:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776945.502678:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776945.502680:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776945.502683:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776945.502690:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502696:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776945.502698:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502699:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.502700:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776945.502704:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502710:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776945.502713:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502720:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776945.502722:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502724:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776945.502725:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502726:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776945.502728:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776945.502730:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776945.502733:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776945.502735:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776945.502738:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776945.502746:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099814784:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776945.502750:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502752:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.502754:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d500 x1862033099814528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502760:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776945.502762:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776945.502764:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775776945.502767:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776945.502768:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776945.502769:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776945.502771:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776945.502773:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b000. 02000000:00000001:2.0:1775776945.502776:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776945.502777:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776945.502802:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d500. 00000100:00000001:2.0:1775776945.502806:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776945.502807:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502808:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.502812:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6df80 x1862033099814656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502817:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776945.502818:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776945.502820:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775776945.502822:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776945.502823:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776945.502824:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776945.502826:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776945.502827:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a000. 02000000:00000001:2.0:1775776945.502830:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776945.502831:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776945.502832:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6df80. 00000100:00000001:2.0:1775776945.502834:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776945.502835:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502837:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776945.502840:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ca80 x1862033099814784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776945.502846:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776945.502847:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776945.502848:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776945.502850:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776945.502851:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776945.502852:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776945.502854:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776945.502855:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39c00. 02000000:00000001:2.0:1775776945.502857:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776945.502858:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776945.502859:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ca80. 00000100:00000001:2.0:1775776945.502861:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776945.502862:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502864:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776945.502869:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776945.502870:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776945.502872:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776945.502873:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776946.203894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776946.203903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776946.203905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776946.203910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776946.203914:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776946.203917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776946.203920:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776946.203923:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776946.203925:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776946.268020:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776946.268035:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776946.268042:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776946.268043:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776946.268046:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776946.268047:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776946.268050:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776946.268051:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776946.268496:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776946.268499:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776946.268502:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776946.396164:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776946.396170:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776946.396173:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776946.396178:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776946.396181:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776946.396185:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776946.396187:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776946.396189:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776946.396192:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776946.779893:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313334517, at 4313334528 00000100:00000010:1.0:1775776946.779910:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000040:0.0:1775776946.780016:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313334517, at 4313334528 00000100:00000010:0.0:1775776946.780048:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000001:3.0:1775776947.227991:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776947.228000:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776947.228002:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776947.228007:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776947.228021:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776947.228026:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776947.228028:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776947.228031:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776947.228034:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776947.291921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776947.291927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776947.291933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776947.291934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776947.291937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776947.291938:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776947.291941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776947.291943:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776947.292540:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776947.292544:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776947.292547:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776947.419898:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776947.419905:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776947.419907:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776947.419912:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776947.419914:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776947.419918:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776947.419920:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776947.419922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776947.419925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776948.251900:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776948.251907:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776948.251908:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776948.251915:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776948.251918:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776948.251922:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776948.251925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776948.251927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776948.251930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776948.315920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776948.315926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776948.315931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776948.315932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776948.315936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776948.315937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776948.315940:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776948.315941:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776948.316012:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776948.316019:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776948.316023:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776948.443902:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776948.443909:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776948.443912:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776948.443917:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776948.443922:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776948.443926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776948.443929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776948.443931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776948.443934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776949.275912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776949.275919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776949.275921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776949.275926:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776949.275930:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776949.275933:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776949.275935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776949.275938:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776949.275941:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776949.340217:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776949.340226:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776949.340229:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776949.340233:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776949.340238:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776949.340240:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776949.340242:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776949.340245:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776949.340250:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776949.340273:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776949.340278:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776949.340279:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776949.340285:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776949.340286:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776949.340377:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776949.340381:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776949.340384:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776949.467909:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776949.467916:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776949.467917:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776949.467921:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776949.467924:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776949.467927:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776949.467929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776949.467931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776949.467933:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776950.299896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776950.299903:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776950.299905:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776950.299910:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776950.299913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776950.299917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776950.299920:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776950.299922:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776950.299924:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776950.363894:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776950.363899:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776950.363904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776950.363906:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776950.363909:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776950.363910:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776950.363913:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776950.363914:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776950.363998:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776950.364001:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776950.364005:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776950.491872:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776950.491884:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776950.491886:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776950.491894:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776950.491898:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776950.491901:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776950.491904:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776950.491907:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776950.491910:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776950.619935:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776950.619946:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776950.619948:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776950.619962:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776950.619967:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776950.619982:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18665 00000400:00000200:2.0:1775776950.619992:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776950.619997:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776950.620000:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776950.620003:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620006:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776950.620009:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620011:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776950.620015:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776950.620017:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776950.620020:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776950.620022:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776950.620025:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776950.620027:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776950.620030:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620032:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776950.620034:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620036:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776950.620039:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776950.620042:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620045:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776950.620047:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776950.620050:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776950.620053:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620055:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776950.620058:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776950.620066:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c000. 00000020:00000040:2.0:1775776950.620070:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776950.620073:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776950.620075:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620078:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776950.620080:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776950.620085:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776950.620092:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505000. 00000100:00000001:2.0:1775776950.620099:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776950.620107:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776950.620118:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776950.620123:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776950.620128:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4798067] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776950.620155:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776950.620158:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776950.620173:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776950.620175:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776950.620177:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776950.620180:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776950.620182:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18665 00000400:00000200:2.0:1775776950.620186:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776950.620190:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776950.620193:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776950.620196:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620199:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776950.620201:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620203:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:0.0:1775776950.620204:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1775776950.620206:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775776950.620209:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1775776950.620209:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776950.620211:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775776950.620213:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1775776950.620214:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776950.620218:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776950.620222:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776950.620223:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776950.620229:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620235:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776950.620239:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776950.620241:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776950.620242:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:0.0:1775776950.620244:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776950.620246:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 02000000:00000001:0.0:1775776950.620249:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1775776950.620250:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 02000000:00000001:0.0:1775776950.620253:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776950.620254:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.620258:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776950.620258:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1775776950.620264:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099814912:192.168.201.139@tcp:250:kworker.0 02000000:00000001:2.0:1775776950.620266:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776950.620270:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776950.620271:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620274:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776950.620275:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776950.620278:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775776950.620278:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776950.620283:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1775776950.620284:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620287:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776950.620289:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 00000100:00000010:2.0:1775776950.620292:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e300. 02000000:00000001:0.0:1775776950.620293:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776950.620296:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776950.620300:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775776950.620301:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:2.0:1775776950.620302:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620305:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1775776950.620306:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 02000000:00000001:2.0:1775776950.620307:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776950.620310:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776950.620315:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 02000000:00000010:2.0:1775776950.620315:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506c00. 00000100:00000001:2.0:1775776950.620320:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776950.620324:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099814912, portal 25 00000100:00100000:2.0:1775776950.620327:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776950.620328:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776950.620331:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288710656 : -114841420840960 : ffff978d65e7c000) 00000100:00000001:2.0:1775776950.620341:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776950.620343:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776950.620346:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776950.620349:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff8c8494] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.620364:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775776950.620370:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776950.620374:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00080000:2.0:1775776950.620374:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1775776950.620379:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00080000:2.0:1775776950.620379:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:0.0:1775776950.620383:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099814912, offset 0 00000100:00000001:2.0:1775776950.620383:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776950.620385:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776950.620389:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776950.620394:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18665 00000400:00000200:0.0:1775776950.620395:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776950.620401:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776950.620405:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000200:0.0:1775776950.620407:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:2.0:1775776950.620408:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776950.620413:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:0.0:1775776950.620417:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:2.0:1775776950.620417:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776950.620422:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:0.0:1775776950.620424:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775776950.620426:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:0.0:1775776950.620431:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000020:00000001:2.0:1775776950.620431:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776950.620434:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000400:00000200:0.0:1775776950.620438:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1775776950.620438:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776950.620442:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000400:00000200:0.0:1775776950.620444:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000020:00000001:2.0:1775776950.620446:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:0.0:1775776950.620450:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4597:4597:256:4294967295] 192.168.201.139@tcp LPNI seq info [4597:4597:8:4294967295] 00000400:00000200:0.0:1775776950.620456:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:2.0:1775776950.620464:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:0.0:1775776950.620469:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000040:2.0:1775776950.620471:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620476:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776950.620480:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000800:00000200:0.0:1775776950.620481:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620485:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000800:00000010:0.0:1775776950.620488:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000020:00000040:2.0:1775776950.620488:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776950.620492:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620495:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776950.620497:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.0:1775776950.620500:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:2.0:1775776950.620501:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1775776950.620505:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 02000000:00000001:2.0:1775776950.620506:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620509:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776950.620512:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000800:00000200:0.1:1775776950.620515:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00000001:2.0:1775776950.620516:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620520:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776950.620520:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000100:00000010:2.0:1775776950.620525:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7fb80. 00000800:00000100:0.0:1775776950.620526:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000020:00000040:2.0:1775776950.620529:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000010:0.0:1775776950.620531:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 02000000:00000001:2.0:1775776950.620533:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776950.620535:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620538:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775776950.620539:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1775776950.620539:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776950.620542:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776950.620547:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 02000000:00000010:2.0:1775776950.620548:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000400:00000200:0.0:1775776950.620552:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000100:00000001:2.0:1775776950.620553:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776950.620556:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00100000:2.0:1775776950.620561:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776950.620564:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776950.620570:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776950.620572:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776950.620577:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.620581:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.620585:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776950.620585:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1787ebf] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.620596:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.620597:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776950.620603:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776950.620610:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776950.620610:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.620615:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313339637 00000100:00080000:2.0:1775776950.620615:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776950.620620:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776950.620622:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776950.620623:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776950.620625:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776950.620628:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18665 00000400:00000200:2.0:1775776950.620632:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776950.620635:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776950.620638:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776950.620640:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620642:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:0.0:1775776950.620644:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000040:2.0:1775776950.620645:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620650:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776950.620652:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776950.620653:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000100:00000001:0.0:1775776950.620654:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1775776950.620655:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776950.620658:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776950.620660:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:0.0:1775776950.620662:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313339637 00000020:00000001:2.0:1775776950.620663:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000010:0.0:1775776950.620667:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f2370c0. 00000100:00000040:2.0:1775776950.620669:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776950.620671:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776950.620673:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776950.620675:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775776950.620678:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776950.620679:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.620684:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1775776950.620684:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776950.620688:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776950.620691:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620693:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776950.620695:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1775776950.620699:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776950/real 1775776950] req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776950.620708:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:0.0:1775776950.620710:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1775776950.620711:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775776950.620713:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1775776950.620715:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:0.0:1775776950.620716:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000008:00100000:2.0:1775776950.620719:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000400:00000200:0.0:1775776950.620720:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000008:00000001:2.0:1775776950.620724:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776950.620725:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:2.0:1775776950.620729:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.620731:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:2.0:1775776950.620734:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e680. 00000100:00000200:0.0:1775776950.620736:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775776950.620737:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776950.620743:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776950.620746:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.620750:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776950.620753:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:0.0:1775776950.620756:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776950.620756:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776950.620762:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506800. 00000100:00000001:0.0:1775776950.620763:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776950.620765:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776950.620767:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.620768:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.620770:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776950.620775:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776950.620776:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776950.620810:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776950.620815:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776950.620840:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776950.620843:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.620848:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1775776950.620849:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086433f13] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.620854:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.620859:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.620861:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.620866:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776950.620867:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.620869:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776950.620870:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776950.620875:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776950.620876:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776950.620879:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776950.620882:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775776950.620884:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776950.620885:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776950.620890:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1775776950.620891:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18665 00000100:00000040:0.0:1775776950.620899:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776950.620899:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776950.620905:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776950.620910:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775776950.620916:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099814912:192.168.201.139@tcp:250:kworker.0 00000100:00000040:2.0:1775776950.620917:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620922:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776950.620925:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776950.620926:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776950.620929:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775776950.620930:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776950.620934:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:0.0:1775776950.620935:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c000 x1862033099814912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775776950.620936:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776950.620939:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776950.620942:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776950.620947:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:0.0:1775776950.620962:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1775776950.620963:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 02000000:00000001:0.0:1775776950.620965:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1775776950.620969:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620971:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776950.620973:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776950.620975:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000010:0.0:1775776950.620977:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 00000020:00000040:2.0:1775776950.620978:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 02000000:00000001:0.0:1775776950.620983:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1775776950.620984:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000020:00000001:0.0:1775776950.620986:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1775776950.620988:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776950.620989:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776950.620991:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:0.0:1775776950.620994:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:2.0:1775776950.620996:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1775776950.620997:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776950.620999:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.621002:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1775776950.621003:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776950.621006:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000008:00100000:2.0:1775776950.621007:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000010:0.0:1775776950.621010:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c000. 00000008:00000001:2.0:1775776950.621011:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.621015:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621016:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776950.621019:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1775776950.621019:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ea00. 00000100:00000001:0.0:1775776950.621023:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775776950.621024:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776950.621029:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776950.621030:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.621032:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776950.621033:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775776950.621034:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1775776950.621035:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621037:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776950.621039:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:0.0:1775776950.621041:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776950.621043:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776950.621044:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776950.621046:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776950.621049:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776950.621055:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776950.621057:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776950.621059:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.621063:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776950.621066:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776950.621070:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776950.621071:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1775776950.621074:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815040:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776950.621075:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621078:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776950.621081:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776950.621083:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776950.621084:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc493d27] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776950.621085:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776950.621088:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776950.621092:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776950.621098:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776950.621102:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000100:00000001:2.0:1775776950.621103:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776950.621104:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776950.621109:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099815040, portal 10 00000100:00000001:0.0:1775776950.621110:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1775776950.621111:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18676) 00000100:00000001:0.0:1775776950.621112:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288719616 : -114841420832000 : ffff978d65e7e300) 00000100:00000040:0.0:1775776950.621119:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621123:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776950.621125:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776950.621127:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776950.621128:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099815040, offset 0 00000400:00000200:0.0:1775776950.621131:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776950.621134:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776950.621136:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776950.621138:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776950.621139:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776950.621141:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776950.621143:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776950.621145:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4598:4598:256:4294967295] 192.168.201.139@tcp LPNI seq info [4598:4598:8:4294967295] 00000400:00000200:0.0:1775776950.621147:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776950.621151:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776950.621153:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776950.621156:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776950.621158:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776950.621160:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775776950.621163:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776950.621165:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775776950.621167:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776950.621169:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776950.621170:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776950.621172:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776950.621174:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776950.621175:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776950.621177:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776950.621179:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621183:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.621185:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621189:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621190:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776950.621193:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776950.621196:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776950.621199:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776950.621201:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313338373 00000100:00000001:0.0:1775776950.621204:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313339637 00000100:00000001:0.0:1775776950.621205:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313338373 00000100:00000001:0.0:1775776950.621207:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313339637 00000100:00000001:0.0:1775776950.621209:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776950.621211:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621212:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776950.621215:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776950/real 1775776950] req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776950.621228:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776950.621229:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776950.621231:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776950.621233:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776950.621235:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776950.621237:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776950.621239:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776950.621247:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621253:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776950.621255:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621256:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.621257:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776950.621260:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621266:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776950.621271:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621277:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776950.621279:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621281:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.621282:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621284:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776950.621285:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776950.621288:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776950.621291:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776950.621293:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776950.621297:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776950.621304:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815040:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776950.621307:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776950.621310:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621316:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776950.621317:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.621319:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776950.621321:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776950.621322:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776950.621325:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815168:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776950.621328:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776950.621329:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776950.621331:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.621332:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776950.621336:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776950.621338:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776950.621341:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776950.621343:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776950.621346:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776950.621348:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099815168, portal 10 00000100:00000001:0.0:1775776950.621350:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776950.621351:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288725888 : -114841420825728 : ffff978d65e7fb80) 00000100:00000040:0.0:1775776950.621356:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621362:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776950.621364:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776950.621366:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776950.621369:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099815168, offset 0 00000400:00000200:0.0:1775776950.621373:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776950.621377:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776950.621379:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776950.621382:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776950.621384:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776950.621386:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776950.621388:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776950.621391:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4599:4599:256:4294967295] 192.168.201.139@tcp LPNI seq info [4599:4599:8:4294967295] 00000400:00000200:0.0:1775776950.621395:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776950.621400:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776950.621404:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776950.621408:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776950.621411:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776950.621413:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775776950.621417:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776950.621419:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775776950.621423:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776950.621424:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776950.621427:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776950.621430:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776950.621433:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776950.621434:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776950.621436:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776950.621439:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621447:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.621449:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621455:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621456:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776950.621458:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776950.621461:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776950.621463:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776950.621466:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313338374 00000100:00000001:0.0:1775776950.621468:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313339637 00000100:00000001:0.0:1775776950.621470:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313338374 00000100:00000001:0.0:1775776950.621472:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313339637 00000100:00000001:0.0:1775776950.621474:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776950.621476:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621477:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776950.621480:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776950/real 1775776950] req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776950.621487:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776950.621488:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776950.621490:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776950.621492:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776950.621493:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776950.621495:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776950.621498:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776950.621506:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621512:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776950.621513:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621514:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.621515:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776950.621518:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621523:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776950.621527:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621532:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776950.621535:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621537:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.621538:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621539:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776950.621541:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776950.621543:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776950.621547:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776950.621549:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776950.621552:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776950.621560:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815168:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776950.621564:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776950.621567:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621573:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776950.621574:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.621577:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776950.621578:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776950.621580:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776950.621583:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815296:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776950.621586:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776950.621588:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776950.621589:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.621591:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776950.621594:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776950.621596:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776950.621599:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776950.621601:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776950.621605:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776950.621608:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099815296, portal 4 00000100:00000001:0.0:1775776950.621610:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776950.621612:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288720512 : -114841420831104 : ffff978d65e7e680) 00000100:00000040:0.0:1775776950.621617:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621623:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776950.621625:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776950.621627:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776950.621629:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099815296, offset 0 00000400:00000200:0.0:1775776950.621633:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776950.621637:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776950.621640:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776950.621642:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776950.621644:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776950.621647:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776950.621649:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776950.621652:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4600:4600:256:4294967295] 192.168.201.139@tcp LPNI seq info [4600:4600:8:4294967295] 00000400:00000200:0.0:1775776950.621655:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776950.621661:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776950.621665:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776950.621668:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776950.621672:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776950.621674:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775776950.621678:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776950.621680:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775776950.621684:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776950.621686:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776950.621689:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776950.621692:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776950.621694:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776950.621696:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776950.621698:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776950.621702:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621714:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.621717:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621723:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621724:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776950.621727:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776950.621730:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776950.621733:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776950.621735:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313338383 00000100:00000001:0.0:1775776950.621737:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313339637 00000100:00000001:0.0:1775776950.621738:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313338383 00000100:00000001:0.0:1775776950.621740:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313339637 00000100:00000001:0.0:1775776950.621742:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776950.621744:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621746:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776950.621749:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776950/real 1775776950] req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776950.621756:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776950.621757:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776950.621758:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776950.621761:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776950.621762:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776950.621764:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776950.621767:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776950.621774:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621801:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776950.621802:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621804:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.621805:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776950.621809:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621815:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776950.621819:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621825:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776950.621827:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621829:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.621830:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621831:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776950.621833:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776950.621835:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776950.621838:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776950.621840:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776950.621843:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776950.621850:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815296:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776950.621854:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621856:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.621859:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e300 x1862033099815040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621864:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776950.621866:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776950.621868:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776950.621871:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776950.621872:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776950.621874:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776950.621875:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776950.621877:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:0.0:1775776950.621880:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776950.621881:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776950.621882:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e300. 00000100:00000001:0.0:1775776950.621884:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776950.621886:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621887:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.621890:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7fb80 x1862033099815168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621896:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776950.621897:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776950.621898:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776950.621901:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776950.621902:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776950.621903:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776950.621905:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776950.621906:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:0.0:1775776950.621908:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776950.621909:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776950.621910:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7fb80. 00000100:00000001:0.0:1775776950.621912:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776950.621913:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621915:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.621918:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e680 x1862033099815296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621923:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776950.621924:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776950.621926:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776950.621928:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776950.621929:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776950.621930:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776950.621932:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776950.621933:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506800. 02000000:00000001:0.0:1775776950.621935:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776950.621937:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776950.621938:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e680. 00000100:00000001:0.0:1775776950.621940:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776950.621941:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621942:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.621946:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776950.621948:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.621950:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776950.621963:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776950.621964:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776950.621968:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.621974:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776950.621975:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.621977:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776950.621979:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776950.621980:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776950.621983:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815424:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776950.621986:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776950.621987:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776950.621989:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776950.621990:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776950.621993:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776950.621995:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776950.621998:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776950.622000:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776950.622003:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776950.622006:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099815424, portal 4 00000100:00000001:0.0:1775776950.622008:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776950.622009:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288721408 : -114841420830208 : ffff978d65e7ea00) 00000100:00000040:0.0:1775776950.622014:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.622019:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776950.622022:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776950.622024:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776950.622026:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099815424, offset 0 00000400:00000200:0.0:1775776950.622031:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776950.622034:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776950.622038:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776950.622040:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776950.622043:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776950.622045:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776950.622047:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776950.622050:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4601:4601:256:4294967295] 192.168.201.139@tcp LPNI seq info [4601:4601:8:4294967295] 00000400:00000200:0.0:1775776950.622053:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776950.622058:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776950.622063:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776950.622065:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776950.622069:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776950.622072:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775776950.622076:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776950.622078:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775776950.622081:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776950.622083:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776950.622085:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776950.622088:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776950.622090:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776950.622092:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776950.622094:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776950.622098:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.622105:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.622108:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776975 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.622114:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.622115:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776950.622119:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776950.622121:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776950.622124:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776950.622126:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313338375 00000100:00000001:0.0:1775776950.622129:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313339637 00000100:00000001:0.0:1775776950.622130:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313338375 00000100:00000001:0.0:1775776950.622132:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313339637 00000100:00000001:0.0:1775776950.622134:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776950.622136:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.622138:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776950.622141:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776950/real 1775776950] req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776950.622149:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776950.622150:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776950.622152:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776950.622154:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776950.622155:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776950.622157:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776950.622160:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776950.622168:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.622174:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776950.622175:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.622177:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.622178:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776950.622182:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.622188:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776950.622192:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.622199:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776950.622201:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.622203:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776950.622204:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.622206:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776950.622207:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776950.622210:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776950.622213:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776950.622215:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776950.622219:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776950.622233:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815424:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776950.622236:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.622237:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776950.622240:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ea00 x1862033099815424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776975 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776950.622246:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776950.622248:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776950.622249:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776950.622252:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776950.622253:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776950.622254:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776950.622256:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776950.622258:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:0.0:1775776950.622260:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776950.622261:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776950.622262:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ea00. 00000100:00000001:0.0:1775776950.622265:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776950.622266:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.622268:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776950.622271:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776950.622272:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776950.622274:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776950.622275:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776951.324053:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776951.324060:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776951.324063:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776951.324069:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776951.324072:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776951.324075:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776951.324078:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776951.324080:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776951.324083:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387909:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776951.387915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776951.387924:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776951.387927:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776951.387930:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387933:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776951.387935:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387965:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387969:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387972:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776951.387974:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776951.387975:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776951.387976:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387978:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776951.387979:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387981:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387982:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387986:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776951.387987:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776951.387988:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776951.387990:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387991:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776951.387992:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387993:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776951.387995:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776951.388061:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776951.388066:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776951.388069:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776951.515893:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776951.515899:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776951.515901:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776951.515905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776951.515907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776951.515911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776951.515912:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776951.515914:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776951.515916:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776951.899881:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313339637, at 4313339648 00000100:00000010:1.0:1775776951.899895:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f2370c0. 00000100:00000040:0.0:1775776951.899897:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313339637, at 4313339648 00000100:00000010:0.0:1775776951.899903:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775776952.347890:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776952.347895:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776952.347897:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776952.347902:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776952.347906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776952.347910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776952.347912:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776952.347915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776952.347917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776952.411910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776952.411915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776952.411932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776952.411933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776952.411936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776952.411938:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776952.411941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776952.411942:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776952.412009:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776952.412013:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776952.412027:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776952.540243:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776952.540249:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776952.540250:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776952.540255:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776952.540261:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776952.540265:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776952.540267:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776952.540269:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776952.540272:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776953.371877:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776953.371883:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776953.371884:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776953.371888:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776953.371891:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776953.371893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776953.371896:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776953.371897:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776953.371899:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776953.435887:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776953.435892:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776953.435896:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776953.435896:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776953.435899:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776953.435900:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776953.435911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776953.435912:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776953.436009:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776953.436013:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776953.436016:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776953.563872:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776953.563877:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776953.563878:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776953.563885:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776953.563888:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776953.563890:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776953.563892:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776953.563893:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776953.563895:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776954.395910:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776954.395918:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776954.395919:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776954.395925:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776954.395928:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776954.395932:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776954.395934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776954.395937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776954.395939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776954.459925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776954.459931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776954.459935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776954.459936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776954.459938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776954.459939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776954.459941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776954.459942:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776954.460005:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776954.460007:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776954.460010:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776954.587924:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776954.587930:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776954.587932:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776954.587937:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776954.587941:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776954.587944:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776954.587946:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776954.587949:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776954.587951:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776955.419901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776955.419909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776955.419911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776955.419916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776955.419920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776955.419924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776955.419927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776955.419929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776955.419932:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776955.484004:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776955.484011:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776955.484016:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776955.484017:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776955.484021:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776955.484022:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776955.484025:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776955.484026:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776955.484106:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776955.484109:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776955.484113:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776955.611997:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776955.612005:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776955.612007:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776955.612013:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776955.612017:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776955.612021:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776955.612024:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776955.612026:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776955.612029:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776955.739908:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776955.739917:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776955.739919:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776955.739923:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776955.739926:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776955.739932:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18671 00000400:00000200:2.0:1775776955.739941:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776955.739946:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776955.739949:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776955.739951:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.739954:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.739955:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.739957:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776955.739960:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776955.739961:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776955.739963:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776955.739965:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776955.739966:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776955.739968:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776955.739969:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.739970:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.739972:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.739973:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776955.739975:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776955.739977:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776955.739978:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776955.739981:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776955.739983:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776955.739984:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.739986:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776955.739987:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776955.739995:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ce00. 00000020:00000040:2.0:1775776955.739997:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776955.740000:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776955.740001:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740004:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776955.740006:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776955.740011:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776955.740019:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507000. 00000100:00000001:2.0:1775776955.740025:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776955.740034:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776955.740044:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776955.740049:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776955.740054:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050a5f9f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776955.740081:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776955.740083:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776955.740086:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776955.740087:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776955.740088:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776955.740090:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776955.740091:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18671 00000400:00000200:2.0:1775776955.740094:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776955.740096:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776955.740098:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776955.740099:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740101:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.740102:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740103:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776955.740105:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776955.740106:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776955.740107:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776955.740108:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776955.740110:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776955.740111:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776955.740113:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740115:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.740116:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740118:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776955.740120:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776955.740121:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776955.740123:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776955.740123:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776955.740126:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776955.740129:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776955.740130:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740132:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776955.740135:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1775776955.740135:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776955.740139:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740140:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776955.740142:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d180. 00000020:00000040:2.0:1775776955.740144:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776955.740145:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775776955.740146:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1775776955.740146:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740147:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776955.740149:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776955.740150:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776955.740154:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775776955.740157:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506000. 00000100:00000001:2.0:1775776955.740162:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.740164:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776955.740166:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776955.740168:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775776955.740170:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776955.740173:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776955.740178:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776955.740180:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776955.740182:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008292136e] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776955.740183:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776955.740191:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099815552:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776955.740197:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776955.740197:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776955.740200:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1775776955.740202:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776955.740205:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776955.740205:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776955.740208:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1775776955.740209:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1775776955.740210:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776955.740212:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776955.740214:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18671 00000400:00000200:2.0:1775776955.740216:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776955.740218:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776955.740219:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776955.740220:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740222:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.740223:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740224:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776955.740225:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776955.740226:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776955.740227:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776955.740228:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776955.740229:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776955.740230:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776955.740231:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740232:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.740233:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740234:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776955.740235:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776955.740237:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740238:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776955.740239:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776955.740240:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776955.740241:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740242:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776955.740243:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776955.740244:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740246:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776955.740248:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ca80. 00000020:00000040:2.0:1775776955.740249:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776955.740250:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776955.740251:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740252:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776955.740253:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776955.740254:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776955.740256:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:2.0:1775776955.740257:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776955.740260:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776955.740264:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776955.740265:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776955.740267:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a559b88] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776955.740271:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776955.740273:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776955.740275:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776955.740276:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776955.740277:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776955.740278:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776955.740279:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18671 00000400:00000200:2.0:1775776955.740281:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776955.740282:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776955.740283:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776955.740285:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740286:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.740287:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740288:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776955.740289:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776955.740290:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776955.740291:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776955.740292:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776955.740293:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776955.740295:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776955.740296:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740297:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.740298:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740299:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776955.740300:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776955.740301:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740302:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776955.740303:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776955.740304:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776955.740305:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740306:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776955.740308:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776955.740309:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740310:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776955.740311:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ea00. 00000020:00000040:2.0:1775776955.740313:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776955.740314:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776955.740314:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740315:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776955.740316:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776955.740316:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776955.740318:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506800. 00000100:00000001:2.0:1775776955.740320:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776955.740322:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776955.740326:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776955.740327:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776955.740329:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc493d27] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776955.740333:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776955.740335:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776955.740336:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776955.740337:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776955.740338:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776955.740339:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776955.740340:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18671 00000400:00000200:2.0:1775776955.740342:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776955.740343:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776955.740344:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776955.740345:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740346:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.740347:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740348:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776955.740349:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776955.740350:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776955.740351:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000010:0.0:1775776955.740352:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 00000020:00000040:2.0:1775776955.740352:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776955.740355:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 02000000:00000001:0.0:1775776955.740356:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776955.740357:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776955.740360:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740361:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776955.740362:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740363:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:0.0:1775776955.740365:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000040:2.0:1775776955.740366:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776955.740368:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000010:0.0:1775776955.740369:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000100:00000001:2.0:1775776955.740370:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776955.740373:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776955.740376:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1775776955.740377:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 02000000:00000001:2.0:1775776955.740377:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740379:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776955.740380:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776955.740381:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740382:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776955.740385:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e680. 00000020:00000040:2.0:1775776955.740388:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776955.740390:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776955.740390:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776955.740391:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776955.740392:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776955.740393:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776955.740394:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099815552, portal 25 02000000:00000010:2.0:1775776955.740395:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:0.0:1775776955.740397:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776955.740398:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.740400:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288714240 : -114841420837376 : ffff978d65e7ce00) 00000100:00100000:2.0:1775776955.740403:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1775776955.740408:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776955.740409:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776955.740412:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776955.740418:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086433f13] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740419:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776955.740426:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00000001:2.0:1775776955.740427:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776955.740430:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18681) 00000400:00000010:0.0:1775776955.740436:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776955.740439:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099815552, offset 0 00000400:00000200:0.0:1775776955.740450:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776955.740457:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.740462:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776955.740465:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776955.740468:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776955.740470:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776955.740472:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776955.740475:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4602:4602:256:4294967295] 192.168.201.139@tcp LPNI seq info [4602:4602:8:4294967295] 00000400:00000200:0.0:1775776955.740478:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.740493:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776955.740497:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.740501:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776955.740505:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776955.740508:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776955.740512:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776955.740514:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776955.740517:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.740518:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776955.740523:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776955.740539:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776955.740542:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776955.740544:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776955.740547:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776955.740550:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740557:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.740559:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740564:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.740565:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776955.740569:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776955.740572:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776955.740574:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313344757 00000100:00000010:0.0:1775776955.740578:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776955.740590:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776955.740594:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313344757 00000100:00000010:0.0:1775776955.740596:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f2373c0. 00000100:00000001:0.0:1775776955.740598:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776955.740603:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776955.740605:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.740607:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776955.740610:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776955/real 1775776955] req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776955.740620:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776955.740621:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776955.740623:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776955.740625:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776955.740626:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776955.740629:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776955.740641:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776955.740650:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740656:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776955.740658:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.740660:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.740662:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776955.740666:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740673:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776955.740677:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740694:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776955.740696:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.740697:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.740698:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.740700:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776955.740701:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776955.740703:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776955.740706:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776955.740707:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776955.740710:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776955.740725:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099815552:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776955.740728:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.740729:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.740731:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ce00 x1862033099815552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740741:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776955.740743:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776955.740745:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776955.740747:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776955.740748:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776955.740749:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776955.740751:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776955.740752:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507000. 02000000:00000001:0.0:1775776955.740755:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776955.740756:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776955.740758:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ce00. 00000100:00000001:0.0:1775776955.740760:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776955.740760:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.740762:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.740767:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776955.740768:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.740770:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776955.740771:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776955.740772:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776955.740774:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740804:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776955.740805:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776955.740806:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776955.740807:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776955.740809:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776955.740811:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815680:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776955.740813:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776955.740814:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776955.740815:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776955.740816:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776955.740818:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776955.740819:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776955.740822:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776955.740824:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776955.740826:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776955.740828:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099815680, portal 10 00000100:00000001:0.0:1775776955.740829:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776955.740830:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288715136 : -114841420836480 : ffff978d65e7d180) 00000100:00000040:0.0:1775776955.740841:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740846:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776955.740847:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776955.740849:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776955.740850:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099815680, offset 0 00000400:00000200:0.0:1775776955.740853:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776955.740856:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.740858:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776955.740883:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776955.740892:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776955.740894:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776955.740895:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776955.740897:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4603:4603:256:4294967295] 192.168.201.139@tcp LPNI seq info [4603:4603:8:4294967295] 00000400:00000200:0.0:1775776955.740900:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.740904:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776955.740907:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.740910:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776955.740912:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776955.740915:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776955.740927:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776955.740930:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776955.740934:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.740944:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776955.740948:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776955.740951:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776955.740954:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776955.740956:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776955.740959:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776955.740962:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740970:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.740972:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.740979:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.740980:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776955.740983:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776955.740994:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776955.740997:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776955.741006:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313343493 00000100:00000001:0.0:1775776955.741009:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313344757 00000100:00000001:0.0:1775776955.741010:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313343493 00000100:00000001:0.0:1775776955.741012:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313344757 00000100:00000001:0.0:1775776955.741013:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776955.741014:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.741015:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776955.741018:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776955/real 1775776955] req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776955.741023:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776955.741024:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776955.741025:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776955.741027:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776955.741028:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776955.741029:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776955.741031:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776955.741035:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741048:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776955.741050:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741051:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.741052:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776955.741056:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741061:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776955.741065:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741072:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776955.741074:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741076:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.741077:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741079:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776955.741081:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776955.741083:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776955.741087:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776955.741104:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776955.741107:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776955.741112:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815680:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776955.741115:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776955.741117:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741121:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776955.741122:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776955.741123:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776955.741124:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776955.741125:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776955.741127:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815808:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776955.741129:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776955.741130:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776955.741131:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776955.741132:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776955.741135:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775776955.741136:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776955.741145:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776955.741146:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776955.741148:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776955.741150:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099815808, portal 10 00000100:00000001:0.0:1775776955.741152:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776955.741152:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288713344 : -114841420838272 : ffff978d65e7ca80) 00000100:00000040:0.0:1775776955.741156:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741160:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776955.741161:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776955.741163:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776955.741164:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099815808, offset 0 00000400:00000200:0.0:1775776955.741166:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776955.741169:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.741171:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776955.741172:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776955.741174:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776955.741176:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776955.741177:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776955.741179:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4604:4604:256:4294967295] 192.168.201.139@tcp LPNI seq info [4604:4604:8:4294967295] 00000400:00000200:0.0:1775776955.741181:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.741185:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776955.741187:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.741195:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776955.741198:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776955.741200:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776955.741202:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776955.741204:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776955.741411:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.741414:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776955.741417:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776955.741419:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776955.741423:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776955.741425:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776955.741427:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776955.741431:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741437:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.741517:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741522:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.741523:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776955.741527:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776955.741529:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776955.741531:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776955.741532:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313343494 00000100:00000001:0.0:1775776955.741534:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313344757 00000100:00000001:0.0:1775776955.741535:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313343494 00000100:00000001:0.0:1775776955.741537:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313344757 00000100:00000001:0.0:1775776955.741538:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776955.741539:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.741540:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776955.741543:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776955/real 1775776955] req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776955.741555:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776955.741556:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776955.741557:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776955.741559:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776955.741560:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776955.741570:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776955.741572:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776955.741577:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741582:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776955.741583:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741584:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.741585:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776955.741587:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741591:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776955.741598:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741603:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776955.741605:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741606:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.741606:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741607:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776955.741608:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776955.741610:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776955.741611:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776955.741612:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776955.741614:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776955.741618:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815808:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776955.741621:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776955.741623:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741628:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776955.741629:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776955.741630:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776955.741632:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776955.741633:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776955.741634:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815936:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776955.741636:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776955.741638:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776955.741638:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776955.741639:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776955.741642:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776955.741643:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776955.741651:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776955.741653:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776955.741655:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776955.741657:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099815936, portal 4 00000100:00000001:0.0:1775776955.741658:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776955.741659:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288721408 : -114841420830208 : ffff978d65e7ea00) 00000100:00000040:0.0:1775776955.741663:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741674:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776955.741676:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776955.741677:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776955.741679:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099815936, offset 0 00000400:00000200:0.0:1775776955.741681:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776955.741684:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.741686:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776955.741688:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776955.741689:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776955.741691:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776955.741693:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776955.741700:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4605:4605:256:4294967295] 192.168.201.139@tcp LPNI seq info [4605:4605:8:4294967295] 00000400:00000200:0.0:1775776955.741703:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.741707:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776955.741710:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.741712:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776955.741714:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776955.741716:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776955.741718:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776955.741720:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776955.741722:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.741723:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776955.741725:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776955.741727:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776955.741729:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776955.741730:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776955.741731:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776955.741733:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741738:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.741739:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741743:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.741744:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776955.741751:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776955.741753:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776955.741755:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776955.741757:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313343503 00000100:00000001:0.0:1775776955.741758:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313344757 00000100:00000001:0.0:1775776955.741759:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313343503 00000100:00000001:0.0:1775776955.741761:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313344757 00000100:00000001:0.0:1775776955.741762:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776955.741763:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.741764:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776955.741773:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776955/real 1775776955] req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776955.741813:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776955.741814:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776955.741816:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776955.741819:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776955.741820:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776955.741822:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776955.741825:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776955.741833:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741839:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776955.741840:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741841:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.741843:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776955.741856:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741873:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776955.741878:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741884:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776955.741887:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741888:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.741889:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.741891:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776955.741893:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776955.741895:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776955.741909:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776955.741911:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776955.741915:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776955.741935:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099815936:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776955.741939:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776955.741943:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.741956:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776955.741958:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776955.741960:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776955.741962:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776955.741963:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776955.741966:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816064:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776955.741969:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776955.741970:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776955.741971:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776955.741973:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776955.741977:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776955.741979:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776955.741982:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776955.741984:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776955.741987:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776955.741990:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099816064, portal 4 00000100:00000001:0.0:1775776955.741992:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776955.741993:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288720512 : -114841420831104 : ffff978d65e7e680) 00000100:00000040:0.0:1775776955.742010:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742017:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776955.742019:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776955.742022:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776955.742024:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099816064, offset 0 00000400:00000200:0.0:1775776955.742029:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776955.742033:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.742036:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776955.742039:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776955.742042:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776955.742044:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776955.742046:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776955.742069:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4606:4606:256:4294967295] 192.168.201.139@tcp LPNI seq info [4606:4606:8:4294967295] 00000400:00000200:0.0:1775776955.742074:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776955.742080:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776955.742085:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.742089:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776955.742092:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776955.742095:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776955.742099:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776955.742114:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776955.742118:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776955.742120:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776955.742123:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776955.742126:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776955.742129:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776955.742130:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776955.742133:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776955.742136:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742143:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.742146:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776980 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742163:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.742164:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776955.742167:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776955.742170:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776955.742173:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776955.742175:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313343495 00000100:00000001:0.0:1775776955.742176:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313344757 00000100:00000001:0.0:1775776955.742178:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313343495 00000100:00000001:0.0:1775776955.742179:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313344757 00000100:00000001:0.0:1775776955.742180:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776955.742182:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.742183:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776955.742185:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776955/real 1775776955] req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776955.742190:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776955.742191:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776955.742192:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776955.742193:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776955.742194:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776955.742196:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776955.742198:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776955.742223:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742230:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776955.742235:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742237:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.742238:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776955.742241:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742247:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776955.742250:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742266:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776955.742268:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742270:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776955.742271:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742272:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776955.742274:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776955.742276:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776955.742279:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776955.742281:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776955.742285:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776955.742293:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816064:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776955.742296:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742298:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.742301:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d180 x1862033099815680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742318:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776955.742319:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776955.742321:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775776955.742324:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776955.742326:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776955.742327:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776955.742329:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776955.742331:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506000. 02000000:00000001:0.0:1775776955.742334:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776955.742336:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776955.742337:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d180. 00000100:00000001:0.0:1775776955.742339:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776955.742340:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742342:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.742345:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ca80 x1862033099815808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742373:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776955.742374:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776955.742375:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775776955.742378:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776955.742379:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776955.742380:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776955.742381:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776955.742382:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:0.0:1775776955.742383:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776955.742384:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776955.742385:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ca80. 00000100:00000001:0.0:1775776955.742387:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776955.742388:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742390:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.742393:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ea00 x1862033099815936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742399:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776955.742400:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776955.742401:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775776955.742412:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776955.742413:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776955.742414:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776955.742416:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776955.742417:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506800. 02000000:00000001:0.0:1775776955.742420:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776955.742421:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776955.742422:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ea00. 00000100:00000001:0.0:1775776955.742424:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776955.742425:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742427:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776955.742430:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e680 x1862033099816064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776980 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776955.742437:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776955.742438:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776955.742439:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776955.742552:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776955.742555:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776955.742566:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776955.742569:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776955.742570:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:0.0:1775776955.742574:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776955.742575:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776955.742576:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e680. 00000100:00000001:0.0:1775776955.742578:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776955.742579:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742581:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776955.742587:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776955.742588:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776955.742590:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776955.742591:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776956.443905:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776956.443912:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776956.443915:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776956.443920:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776956.443922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776956.443925:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776956.443927:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776956.443929:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776956.443931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776956.507941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776956.507947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776956.507953:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776956.507954:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776956.507957:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776956.507959:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776956.507962:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776956.507963:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776956.508029:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776956.508033:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776956.508037:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776956.635922:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776956.635929:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776956.635930:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776956.635935:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776956.635938:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776956.635942:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776956.635945:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776956.635947:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776956.635950:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776957.019912:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313344757, at 4313344768 00000100:00000010:1.0:1775776957.019926:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f2373c0. 00000100:00000040:0.0:1775776957.019932:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313344757, at 4313344768 00000100:00000010:0.0:1775776957.019942:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775776957.467918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776957.467930:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776957.467932:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776957.467938:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776957.467942:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776957.467946:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776957.467948:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776957.467950:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776957.467953:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776957.531966:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776957.531971:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776957.531976:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776957.531977:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776957.531979:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776957.531980:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776957.531982:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776957.531982:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776957.532045:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776957.532056:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776957.532059:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776957.659938:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776957.659946:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776957.659948:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776957.659956:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776957.659960:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776957.659966:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776957.659979:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776957.659983:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776957.659986:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776958.492442:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776958.492449:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776958.492451:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776958.492456:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776958.492460:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776958.492463:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776958.492466:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776958.492468:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776958.492471:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776958.555915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776958.555929:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776958.555932:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776958.555936:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776958.555940:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776958.555942:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776958.555944:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776958.555946:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776958.555950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776958.555952:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776958.555955:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776958.555957:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776958.555960:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776958.555961:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776958.556047:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776958.556051:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776958.556054:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776958.685187:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776958.685194:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776958.685196:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776958.685201:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776958.685204:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776958.685207:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776958.685210:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776958.685212:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776958.685214:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776959.516889:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776959.516896:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776959.516898:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776959.516903:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776959.516906:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776959.516910:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776959.516913:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776959.516915:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776959.516917:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776959.580605:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776959.580610:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776959.580616:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776959.580617:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776959.580620:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776959.580621:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776959.580625:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776959.580626:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776959.580686:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776959.580690:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776959.580693:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776959.709033:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776959.709040:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776959.709042:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776959.709048:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776959.709051:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776959.709055:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776959.709057:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776959.709059:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776959.709062:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776960.539900:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776960.539909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776960.539912:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776960.539917:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776960.539920:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776960.539924:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776960.539926:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776960.539928:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776960.539931:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604034:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776960.604041:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604048:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776960.604052:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776960.604055:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776960.604058:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604061:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776960.604063:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604065:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604074:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604079:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776960.604080:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776960.604082:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776960.604083:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604085:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776960.604086:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604087:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604089:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604092:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776960.604094:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776960.604095:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776960.604097:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604098:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776960.604099:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604100:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776960.604102:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776960.604164:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776960.604168:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776960.604171:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776960.732001:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776960.732008:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776960.732009:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776960.732015:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776960.732018:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776960.732022:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776960.732024:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776960.732026:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776960.732029:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776960.859911:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776960.859918:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776960.859920:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776960.859923:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776960.859926:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776960.859931:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18676 00000400:00000200:0.0:1775776960.859939:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776960.859944:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776960.859946:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776960.859949:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.859953:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776960.859955:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.859957:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776960.859961:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776960.859963:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776960.859965:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776960.859968:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776960.859970:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776960.859972:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776960.859975:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.859977:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776960.859979:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.859981:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776960.859984:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776960.859987:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776960.859989:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776960.859992:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776960.859995:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776960.859998:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860000:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776960.860002:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776960.860010:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776960.860014:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776960.860017:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776960.860018:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860020:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776960.860022:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776960.860026:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776960.860035:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776960.860041:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776960.860048:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776960.860054:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776960.860057:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776960.860060:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776960.860076:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776960.860078:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776960.860080:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776960.860081:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776960.860082:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776960.860083:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776960.860085:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18676 00000400:00000200:0.0:1775776960.860088:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776960.860091:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776960.860093:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776960.860095:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860096:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776960.860098:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860099:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776960.860100:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775776960.860101:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776960.860102:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776960.860104:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776960.860105:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000100:00000001:2.0:1775776960.860105:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775776960.860108:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776960.860108:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775776960.860111:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776960.860115:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860117:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776960.860117:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776960.860121:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860124:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776960.860128:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776960.860133:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776960.860134:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776960.860135:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860136:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776960.860138:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776960.860140:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776960.860142:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776960.860143:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776960.860145:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.860146:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860147:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776960.860151:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775776960.860151:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099816192:192.168.201.139@tcp:250:kworker.0 00000008:00000001:0.0:1775776960.860155:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860157:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776960.860158:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776960.860161:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 02000000:00000001:2.0:1775776960.860161:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1775776960.860162:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776960.860163:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776960.860165:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776960.860167:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.860167:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776960.860169:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776960.860170:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776960.860172:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776960.860172:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776960.860176:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776960.860177:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776960.860180:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776960.860181:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776960.860185:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000100:00100000:0.0:1775776960.860186:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776960.860193:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1775776960.860193:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000001:0.0:1775776960.860196:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776960.860199:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776960.860200:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099816192, portal 25 00000100:00000001:2.0:1775776960.860204:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776960.860206:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741750784 : -114841967800832 : ffff978d454dce00) 00000100:00000001:0.0:1775776960.860208:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776960.860210:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776960.860214:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776960.860215:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776960.860215:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776960.860217:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776960.860220:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18676 00000100:00000040:2.0:1775776960.860220:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776960.860228:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776960.860234:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00000001:2.0:1775776960.860236:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1775776960.860238:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776960.860240:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776960.860244:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000040:0.0:1775776960.860246:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000200:2.0:1775776960.860248:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099816192, offset 0 00000100:00000001:0.0:1775776960.860250:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776960.860254:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776960.860256:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860259:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776960.860262:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776960.860264:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000400:00000200:2.0:1775776960.860265:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:0.0:1775776960.860267:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776960.860270:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000400:00000200:2.0:1775776960.860272:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000020:00000001:0.0:1775776960.860273:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776960.860277:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776960.860279:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1775776960.860281:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860284:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776960.860285:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000040:0.0:1775776960.860287:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860290:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776960.860290:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776960.860294:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776960.860298:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4607:4607:256:4294967295] 192.168.201.139@tcp LPNI seq info [4607:4607:8:4294967295] 00000020:00000040:0.0:1775776960.860302:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:2.0:1775776960.860302:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:0.0:1775776960.860308:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860311:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776960.860315:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1775776960.860319:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 02000000:00000001:0.0:1775776960.860320:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776960.860325:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860327:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776960.860330:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776960.860333:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860334:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776960.860335:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000010:0.0:1775776960.860338:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776960.860342:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000010:2.0:1775776960.860342:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 02000000:00000001:0.0:1775776960.860347:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776960.860349:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860352:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.0:1775776960.860353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:0.0:1775776960.860355:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776960.860358:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776960.860358:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 02000000:00000010:0.0:1775776960.860363:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000800:00000200:2.1:1775776960.860367:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776960.860368:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776960.860372:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000100:00100000:0.0:1775776960.860375:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775776960.860380:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.860383:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776960.860387:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775776960.860392:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776960.860395:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776960.860397:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1775776960.860400:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776960.860404:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000100:00000001:0.0:1775776960.860406:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776960.860408:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00080000:0.0:1775776960.860409:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776960.860413:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776960.860414:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776960.860415:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776960.860415:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775776960.860419:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776960.860422:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18676 00000100:00000200:2.0:1775776960.860424:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776960.860426:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776960.860434:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776960.860436:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776960.860438:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776960.860438:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775776960.860442:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776960.860445:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776960.860446:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860451:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776960.860456:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776960.860458:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776960.860461:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776960.860461:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776960.860465:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000100:00000001:2.0:1775776960.860465:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1775776960.860468:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776960.860470:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776960.860471:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000400:00000200:2.0:1775776960.860477:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:0.0:1775776960.860478:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776960.860481:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313349877 00000100:00000001:0.0:1775776960.860482:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776960.860485:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000010:2.0:1775776960.860486:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc60. 00000100:00000001:0.0:1775776960.860490:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776960.860493:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000001:2.0:1775776960.860493:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1775776960.860497:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860499:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860500:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313349877 02000000:00000001:0.0:1775776960.860501:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776960.860504:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1775776960.860504:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bcc0. 02000000:00000001:0.0:1775776960.860508:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860509:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776960.860510:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776960.860513:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1775776960.860514:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775776960.860517:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860518:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860519:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860521:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000010:0.0:1775776960.860523:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776960.860526:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000100:00100000:2.0:1775776960.860527:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776960/real 1775776960] req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776960.860530:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776960.860533:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860538:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776960.860540:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:2.0:1775776960.860540:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776960.860542:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1775776960.860543:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776960.860544:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776960.860547:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776960.860548:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 02000000:00000010:0.0:1775776960.860549:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:2.0:1775776960.860550:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776960.860554:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776960.860555:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776960.860563:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1775776960.860564:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776960.860591:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776960.860591:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776960.860594:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860595:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776960.860599:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.860599:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.860603:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776960.860610:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776960.860614:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775776960.860614:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776960.860620:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776960.860621:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776960.860624:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776960.860627:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776960.860631:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18676 00000100:00000001:2.0:1775776960.860631:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776960.860636:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:2.0:1775776960.860640:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776960.860641:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776960.860645:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776960.860649:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860653:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776960.860656:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860661:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776960.860661:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1775776960.860662:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776960.860664:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776960.860667:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776960.860667:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1775776960.860668:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776960.860672:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:2.0:1775776960.860672:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1775776960.860676:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:2.0:1775776960.860676:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776960.860680:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776960.860681:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776960.860682:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776960.860685:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776960.860685:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776960.860688:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776960.860690:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776960.860691:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776960.860693:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776960.860697:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776960.860700:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 02000000:00000001:0.0:1775776960.860701:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776960.860705:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776960.860705:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1775776960.860709:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860714:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:2.0:1775776960.860714:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775776960.860716:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776960.860722:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860726:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776960.860733:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000100:00100000:2.0:1775776960.860733:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099816192:192.168.201.139@tcp:250:kworker.0 00000020:00000040:0.0:1775776960.860739:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:2.0:1775776960.860742:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775776960.860743:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775776960.860744:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1775776960.860745:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860749:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:2.0:1775776960.860749:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dce00 x1862033099816192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776960.860752:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776960.860756:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860760:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000010:0.0:1775776960.860762:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 02000000:00000001:2.0:1775776960.860763:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1775776960.860767:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776960.860767:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776960.860805:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776960.860808:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776960.860809:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776960.860812:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776960.860819:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f400. 00000100:00100000:0.0:1775776960.860823:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1775776960.860824:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776960.860827:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776960.860830:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dce00. 00000100:00000001:2.0:1775776960.860835:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776960.860836:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776960.860837:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776960.860840:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860841:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776960.860846:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.860851:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776960.860854:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776960.860855:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.860858:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776960.860861:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775776960.860862:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18686) 00000100:00000001:2.0:1775776960.860863:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776960.860869:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.860877:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776960.860879:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.860881:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776960.860883:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776960.860885:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776960.860888:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816320:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776960.860891:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776960.860893:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776960.860894:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.860896:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776960.860899:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776960.860901:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776960.860905:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776960.860908:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776960.860911:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776960.860915:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099816320, portal 10 00000100:00000001:2.0:1775776960.860917:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776960.860918:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748992 : -114841967802624 : ffff978d454dc700) 00000100:00000040:2.0:1775776960.860925:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.860931:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776960.860933:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776960.860935:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776960.860938:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099816320, offset 0 00000400:00000200:2.0:1775776960.860942:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776960.860946:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776960.860950:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776960.860952:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776960.860955:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776960.860957:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776960.860959:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776960.860962:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4608:4608:256:4294967295] 192.168.201.139@tcp LPNI seq info [4608:4608:8:4294967295] 00000400:00000200:2.0:1775776960.860966:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776960.860971:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776960.860976:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.860979:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776960.860983:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776960.860986:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776960.860990:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776960.860992:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775776960.860996:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.860997:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776960.861000:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776960.861003:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776960.861006:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776960.861008:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776960.861010:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776960.861014:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861021:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776960.861024:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861036:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.861037:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776960.861040:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776960.861043:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776960.861046:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776960.861047:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313348613 00000100:00000001:2.0:1775776960.861050:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313349877 00000100:00000001:2.0:1775776960.861052:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313348613 00000100:00000001:2.0:1775776960.861054:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313349877 00000100:00000001:2.0:1775776960.861056:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776960.861058:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.861059:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776960.861063:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776960/real 1775776960] req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776960.861071:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776960.861072:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776960.861073:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776960.861075:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776960.861077:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776960.861079:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776960.861082:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776960.861090:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861096:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776960.861097:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861098:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.861100:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776960.861104:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861111:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776960.861114:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861120:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776960.861123:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861125:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.861126:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861127:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776960.861130:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776960.861132:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776960.861135:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776960.861138:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776960.861141:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776960.861149:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816320:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776960.861152:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776960.861155:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861162:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776960.861163:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.861165:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776960.861167:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776960.861169:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776960.861171:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816448:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776960.861174:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776960.861177:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776960.861178:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.861180:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776960.861183:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776960.861185:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776960.861188:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776960.861190:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776960.861193:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776960.861196:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099816448, portal 10 00000100:00000001:2.0:1775776960.861198:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776960.861199:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741753472 : -114841967798144 : ffff978d454dd880) 00000100:00000040:2.0:1775776960.861205:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861211:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776960.861213:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776960.861216:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776960.861218:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099816448, offset 0 00000400:00000200:2.0:1775776960.861222:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776960.861226:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776960.861229:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776960.861232:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776960.861234:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776960.861236:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776960.861238:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776960.861241:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4609:4609:256:4294967295] 192.168.201.139@tcp LPNI seq info [4609:4609:8:4294967295] 00000400:00000200:2.0:1775776960.861245:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776960.861251:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776960.861255:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.861259:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776960.861262:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776960.861265:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776960.861269:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776960.861271:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775776960.861275:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.861277:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776960.861279:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776960.861283:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776960.861286:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776960.861287:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776960.861289:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776960.861292:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861299:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776960.861302:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861308:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.861310:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776960.861313:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776960.861315:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776960.861318:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776960.861320:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313348614 00000100:00000001:2.0:1775776960.861322:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313349877 00000100:00000001:2.0:1775776960.861324:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313348614 00000100:00000001:2.0:1775776960.861326:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313349877 00000100:00000001:2.0:1775776960.861328:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776960.861330:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.861332:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776960.861335:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776960/real 1775776960] req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776960.861343:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776960.861344:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776960.861346:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776960.861348:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776960.861350:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776960.861352:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776960.861355:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776960.861363:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861369:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776960.861371:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861373:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.861374:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776960.861377:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861384:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776960.861387:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861393:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776960.861395:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861396:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.861397:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861399:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776960.861400:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776960.861403:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776960.861406:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776960.861408:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776960.861412:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776960.861420:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816448:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776960.861424:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776960.861427:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861434:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776960.861435:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.861437:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776960.861439:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776960.861441:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776960.861444:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816576:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776960.861447:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776960.861449:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776960.861450:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.861452:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776960.861455:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776960.861458:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776960.861461:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776960.861463:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776960.861466:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776960.861469:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099816576, portal 4 00000100:00000001:2.0:1775776960.861471:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776960.861472:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741760640 : -114841967790976 : ffff978d454df480) 00000100:00000040:2.0:1775776960.861478:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861484:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776960.861486:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776960.861489:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776960.861491:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099816576, offset 0 00000400:00000200:2.0:1775776960.861495:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776960.861499:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776960.861503:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776960.861505:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776960.861508:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776960.861510:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776960.861512:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776960.861515:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4610:4610:256:4294967295] 192.168.201.139@tcp LPNI seq info [4610:4610:8:4294967295] 00000400:00000200:2.0:1775776960.861519:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776960.861525:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776960.861529:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.861533:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776960.861537:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776960.861540:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776960.861543:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776960.861546:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775776960.861549:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.861551:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776960.861554:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776960.861557:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776960.861559:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776960.861561:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776960.861563:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776960.861566:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861573:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776960.861576:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861582:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.861584:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776960.861586:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776960.861590:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776960.861592:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776960.861594:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313348623 00000100:00000001:2.0:1775776960.861596:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313349877 00000100:00000001:2.0:1775776960.861598:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313348623 00000100:00000001:2.0:1775776960.861600:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313349877 00000100:00000001:2.0:1775776960.861602:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776960.861605:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.861607:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776960.861611:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776960/real 1775776960] req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776960.861619:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776960.861620:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776960.861622:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776960.861624:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776960.861625:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776960.861627:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776960.861630:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776960.861637:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861643:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776960.861644:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861646:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.861647:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776960.861650:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861657:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776960.861660:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861667:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776960.861669:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861670:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.861672:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861673:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776960.861675:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776960.861677:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776960.861681:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776960.861683:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776960.861686:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776960.861694:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816576:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776960.861697:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776960.861700:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861706:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776960.861707:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.861709:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776960.861711:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776960.861712:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776960.861715:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816704:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776960.861717:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776960.861719:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776960.861720:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776960.861722:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776960.861725:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776960.861727:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776960.861729:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776960.861731:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776960.861734:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776960.861737:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099816704, portal 4 00000100:00000001:2.0:1775776960.861739:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776960.861740:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741758848 : -114841967792768 : ffff978d454ded80) 00000100:00000040:2.0:1775776960.861745:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861751:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776960.861753:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776960.861755:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776960.861758:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099816704, offset 0 00000400:00000200:2.0:1775776960.861761:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776960.861765:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776960.861768:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776960.861771:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776960.861805:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776960.861807:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776960.861809:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776960.861812:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4611:4611:256:4294967295] 192.168.201.139@tcp LPNI seq info [4611:4611:8:4294967295] 00000400:00000200:2.0:1775776960.861816:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776960.861822:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776960.861826:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.861829:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776960.861833:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776960.861835:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775776960.861839:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776960.861842:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775776960.861845:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776960.861847:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776960.861849:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776960.861852:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776960.861855:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776960.861856:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776960.861859:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776960.861862:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861868:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776960.861871:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776985 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861877:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.861878:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776960.861881:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776960.861883:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776960.861886:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776960.861888:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313348615 00000100:00000001:2.0:1775776960.861890:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313349877 00000100:00000001:2.0:1775776960.861892:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313348615 00000100:00000001:2.0:1775776960.861894:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313349877 00000100:00000001:2.0:1775776960.861895:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776960.861897:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.861899:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776960.861902:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776960/real 1775776960] req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776960.861910:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776960.861911:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776960.861912:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776960.861915:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776960.861916:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776960.861918:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776960.861921:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776960.861929:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861935:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776960.861936:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861938:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.861939:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776960.861942:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861948:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776960.861952:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861959:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776960.861961:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861962:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776960.861964:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861965:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776960.861967:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776960.861969:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776960.861972:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776960.861974:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776960.861977:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776960.861984:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816704:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776960.861988:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.861990:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776960.861992:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc700 x1862033099816320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.861998:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776960.862000:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776960.862001:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505000. 02000000:00000001:2.0:1775776960.862004:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776960.862005:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776960.862006:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776960.862008:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776960.862010:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:2.0:1775776960.862012:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776960.862013:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776960.862015:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc700. 00000100:00000001:2.0:1775776960.862017:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776960.862018:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.862020:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776960.862023:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd880 x1862033099816448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.862029:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776960.862030:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776960.862031:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776960.862033:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776960.862034:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776960.862035:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776960.862037:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776960.862038:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:2.0:1775776960.862040:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776960.862041:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776960.862042:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd880. 00000100:00000001:2.0:1775776960.862044:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776960.862045:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.862047:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776960.862049:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df480 x1862033099816576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.862056:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776960.862057:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776960.862058:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506400. 02000000:00000001:2.0:1775776960.862060:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776960.862061:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776960.862062:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776960.862063:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776960.862065:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775776960.862067:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776960.862068:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776960.862069:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df480. 00000100:00000001:2.0:1775776960.862071:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776960.862072:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.862074:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776960.862077:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ded80 x1862033099816704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776985 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776960.862087:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776960.862088:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776960.862090:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775776960.862092:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776960.862093:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776960.862094:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776960.862096:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776960.862097:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:2.0:1775776960.862099:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776960.862100:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776960.862101:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776960.862103:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776960.862104:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.862106:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776960.862110:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776960.862111:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776960.862113:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776960.862114:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776961.563893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776961.563899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776961.563901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776961.563906:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776961.563910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776961.563913:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776961.563915:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776961.563919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776961.563921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776961.627901:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776961.627908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776961.627915:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776961.627916:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776961.627919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776961.627920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776961.627923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776961.627923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776961.627986:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776961.627990:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776961.627993:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776961.755889:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776961.755895:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776961.755897:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776961.755901:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776961.755904:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776961.755906:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776961.755908:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776961.755910:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776961.755912:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776962.140043:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313349877, at 4313349888 00000100:00000010:0.0:1775776962.140058:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000040:1.0:1775776962.140156:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313349877, at 4313349888 00000100:00000010:1.0:1775776962.140162:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:3.0:1775776962.587924:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776962.587931:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776962.587933:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776962.587938:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776962.587942:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776962.587946:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776962.587949:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776962.587951:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776962.587954:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776962.651913:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776962.651923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776962.651929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776962.651930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776962.651934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776962.651935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776962.651938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776962.651939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776962.652006:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776962.652009:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776962.652012:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776962.779901:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776962.779909:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776962.779911:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776962.779916:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776962.779920:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776962.779923:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776962.779925:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776962.779928:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776962.779931:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776963.612170:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776963.612176:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776963.612177:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776963.612182:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776963.612188:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776963.612192:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776963.612195:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776963.612197:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776963.612199:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776963.676545:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776963.676550:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776963.676556:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776963.676557:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776963.676561:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776963.676562:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776963.676565:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776963.676566:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776963.676615:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776963.676618:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776963.676621:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776963.803987:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776963.803995:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776963.803997:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776963.804003:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776963.804006:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776963.804010:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776963.804013:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776963.804015:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776963.804018:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776964.635894:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776964.635902:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776964.635904:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776964.635909:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776964.635913:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776964.635916:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776964.635918:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776964.635920:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776964.635922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776964.699920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776964.699926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776964.699932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776964.699933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776964.699937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776964.699938:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776964.699941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776964.699942:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776964.700009:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776964.700012:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776964.700015:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776964.827920:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776964.827928:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776964.827929:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776964.827942:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776964.827946:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776964.827950:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776964.827952:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776964.827954:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776964.827957:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776965.659912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776965.659919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776965.659921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776965.659926:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776965.659930:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776965.659933:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776965.659935:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776965.659937:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776965.659940:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776965.725957:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776965.725963:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776965.725969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776965.725970:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776965.725973:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776965.725974:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776965.725977:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776965.725977:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776965.726724:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776965.726728:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776965.726732:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776965.851871:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776965.851880:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776965.851883:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776965.851889:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776965.851893:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776965.851897:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776965.851899:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776965.851901:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776965.851904:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776965.979927:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776965.979937:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776965.979940:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776965.979943:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776965.979947:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776965.979954:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18681 00000400:00000200:2.0:1775776965.979963:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776965.979970:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776965.979973:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776965.979976:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.979979:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776965.979981:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.979984:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776965.979988:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776965.979989:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776965.979992:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776965.979994:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776965.979997:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776965.979999:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776965.980011:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980013:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776965.980015:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980017:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776965.980019:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776965.980023:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980025:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776965.980028:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776965.980031:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776965.980033:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980036:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776965.980038:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776965.980048:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7fb80. 00000020:00000040:2.0:1775776965.980051:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776965.980055:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776965.980056:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980059:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776965.980061:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776965.980065:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776965.980073:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505400. 00000100:00000001:2.0:1775776965.980079:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776965.980095:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776965.980109:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776965.980113:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776965.980118:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1787ebf] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776965.980143:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776965.980147:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776965.980151:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776965.980152:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776965.980154:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776965.980157:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776965.980160:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18681 00000400:00000200:2.0:1775776965.980164:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776965.980168:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776965.980171:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776965.980174:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775776965.980174:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980179:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776965.980180:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775776965.980181:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776965.980184:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776965.980185:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776965.980188:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776965.980190:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776965.980193:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:0.0:1775776965.980195:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775776965.980196:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776965.980199:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776965.980203:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776965.980210:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776965.980213:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776965.980214:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776965.980217:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776965.980218:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980222:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:0.0:1775776965.980224:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:2.0:1775776965.980225:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 02000000:00000001:0.0:1775776965.980227:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776965.980230:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776965.980231:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00100000:0.0:1775776965.980235:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099816832:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776965.980236:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776965.980239:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776965.980245:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776965.980247:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980249:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776965.980253:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776965.980256:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980258:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776965.980262:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e300. 00000020:00000040:2.0:1775776965.980264:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776965.980267:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776965.980268:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980269:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776965.980270:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776965.980272:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776965.980276:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505000. 00000100:00000001:2.0:1775776965.980279:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776965.980284:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776965.980291:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776965.980293:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776965.980297:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff8c8494] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776965.980305:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776965.980307:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776965.980311:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776965.980312:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776965.980313:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776965.980315:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776965.980318:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18681 00000400:00000200:2.0:1775776965.980321:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776965.980324:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776965.980326:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776965.980329:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980331:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776965.980333:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980336:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:0.0:1775776965.980338:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1775776965.980339:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776965.980342:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 02000000:00000001:0.0:1775776965.980343:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1775776965.980344:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776965.980347:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 02000000:00000001:0.0:1775776965.980348:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776965.980350:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 02000000:00000001:0.0:1775776965.980352:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1775776965.980354:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776965.980358:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000010:0.0:1775776965.980359:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d400. 00000100:00000001:2.0:1775776965.980360:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776965.980364:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776965.980365:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980369:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:0.0:1775776965.980373:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000040:2.0:1775776965.980373:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000010:0.0:1775776965.980378:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000100:00080000:2.0:1775776965.980378:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980382:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776965.980384:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776965.980387:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776965.980390:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776965.980392:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000001:2.0:1775776965.980393:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776965.980396:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776965.980399:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980401:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776965.980403:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099816832, portal 25 00000100:00000010:2.0:1775776965.980405:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c000. 00000100:00000001:0.0:1775776965.980407:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776965.980409:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288725888 : -114841420825728 : ffff978d65e7fb80) 00000020:00000040:2.0:1775776965.980409:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776965.980415:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776965.980417:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776965.980420:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776965.980420:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776965.980422:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776965.980425:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776965.980431:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:0.0:1775776965.980433:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775776965.980437:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776965.980439:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776965.980443:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00100000:2.0:1775776965.980445:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775776965.980448:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099816832, offset 0 00000100:00000001:2.0:1775776965.980458:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775776965.980460:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980461:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776965.980467:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4798067] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776965.980472:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.980481:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:2.0:1775776965.980481:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776965.980486:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776965.980488:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1775776965.980492:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776965.980495:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775776965.980496:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00080000:2.0:1775776965.980497:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775776965.980502:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1775776965.980503:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775776965.980506:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00080000:2.0:1775776965.980508:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18681 00000400:00000200:0.0:1775776965.980514:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4612:4612:256:4294967295] 192.168.201.139@tcp LPNI seq info [4612:4612:8:4294967295] 00000400:00000200:2.0:1775776965.980516:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.980523:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776965.980524:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776965.980528:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776965.980532:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980537:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776965.980540:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000040:2.0:1775776965.980541:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980546:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776965.980549:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776965.980551:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000800:00000200:0.0:1775776965.980552:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000020:00000001:2.0:1775776965.980554:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776965.980557:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776965.980559:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000800:00000010:0.0:1775776965.980560:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000020:00000001:2.0:1775776965.980563:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776965.980569:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000800:00000200:0.0:1775776965.980573:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776965.980573:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1775776965.980579:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000040:2.0:1775776965.980579:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776965.980582:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776965.980587:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000800:00000200:0.1:1775776965.980591:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00080000:2.0:1775776965.980593:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000800:00000200:0.1:1775776965.980596:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1775776965.980598:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776965.980602:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:0.0:1775776965.980606:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 02000000:00000001:2.0:1775776965.980607:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776965.980610:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776965.980611:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000100:00000001:2.0:1775776965.980613:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776965.980616:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000400:00000200:0.0:1775776965.980619:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:2.0:1775776965.980619:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980624:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776965.980628:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000010:2.0:1775776965.980629:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d500. 00000020:00000040:2.0:1775776965.980632:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 00000400:00000200:0.0:1775776965.980634:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 02000000:00000001:2.0:1775776965.980635:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1775776965.980637:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 02000000:00000001:2.0:1775776965.980638:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980642:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775776965.980644:0:301888:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1775776965.980644:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776965.980646:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776965.980651:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775776965.980651:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775776965.980656:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776965.980666:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776965.980668:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.980675:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776965.980678:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776965.980682:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.980709:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.980711:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776965.980718:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776965.980726:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:2.0:1775776965.980726:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000855db814] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.980729:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313354997 00000100:00000010:0.0:1775776965.980737:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:2.0:1775776965.980738:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776965.980742:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776965.980745:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776965.980753:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313354997 00000100:00000010:0.0:1775776965.980757:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237900. 00000100:00080000:2.0:1775776965.980758:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776965.980761:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776965.980761:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776965.980764:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776965.980766:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776965.980768:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776965.980772:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18681 00000400:00000200:2.0:1775776965.980776:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776965.980799:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.980802:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776965.980807:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776965/real 1775776965] req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776965.980825:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1775776965.980825:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000400:00000001:0.0:1775776965.980829:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1775776965.980829:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1775776965.980832:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000100:00000040:2.0:1775776965.980835:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:0.0:1775776965.980837:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000100:00000001:2.0:1775776965.980839:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1775776965.980840:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000040:2.0:1775776965.980843:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776965.980845:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1775776965.980847:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000200:0.0:1775776965.980851:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776965.980851:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776965.980853:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776965.980857:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776965.980860:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776965.980865:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000200:0.0:1775776965.980866:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776965.980869:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776965.980877:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776965.980879:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776965.980880:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776965.980883:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776965.980884:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776965.980887:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776965.980887:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:0.0:1775776965.980890:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1775776965.980891:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776965.980896:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00100000:0.0:1775776965.980898:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776965.980901:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776965.980905:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776965.980909:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776965.980913:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775776965.980913:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980918:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776965.980921:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000040:0.0:1775776965.980922:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775776965.980924:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776965.980929:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776965.980936:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f100. 00000100:00000001:0.0:1775776965.980937:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1775776965.980939:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:0.0:1775776965.980942:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1775776965.980943:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775776965.980945:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1775776965.980946:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.980948:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776965.980949:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775776965.980952:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1775776965.980952:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775776965.980955:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1775776965.980956:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776965.980961:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000010:2.0:1775776965.980963:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504c00. 00000100:00080000:0.0:1775776965.980967:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776965.980968:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776965.980974:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776965.980977:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1775776965.980985:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1775776965.980987:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776965.980989:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776965.980997:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048605922] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776965.981007:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776965.981011:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099816832:192.168.201.139@tcp:250:kworker.0 00000100:00000040:2.0:1775776965.981013:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18691) 00000100:00000001:0.0:1775776965.981020:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981023:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981027:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7fb80 x1862033099816832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981034:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776965.981036:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981038:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776965.981041:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776965.981042:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776965.981043:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775776965.981044:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776965.981046:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505400. 02000000:00000001:0.0:1775776965.981049:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776965.981050:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776965.981051:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7fb80. 00000100:00000001:0.0:1775776965.981053:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776965.981054:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981055:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981060:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776965.981061:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981062:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776965.981063:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776965.981064:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776965.981067:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981071:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776965.981072:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776965.981073:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776965.981074:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776965.981076:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776965.981077:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816960:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776965.981080:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776965.981081:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776965.981082:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776965.981083:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981085:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776965.981087:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776965.981090:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776965.981092:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776965.981096:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776965.981099:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099816960, portal 10 00000100:00000001:0.0:1775776965.981101:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776965.981102:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288719616 : -114841420832000 : ffff978d65e7e300) 00000100:00000040:0.0:1775776965.981108:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981114:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776965.981116:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776965.981118:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776965.981120:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099816960, offset 0 00000400:00000200:0.0:1775776965.981124:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776965.981129:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.981132:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776965.981134:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776965.981135:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776965.981137:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776965.981138:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776965.981140:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4613:4613:256:4294967295] 192.168.201.139@tcp LPNI seq info [4613:4613:8:4294967295] 00000400:00000200:0.0:1775776965.981143:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.981147:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776965.981151:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776965.981155:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776965.981158:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776965.981161:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776965.981165:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776965.981167:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776965.981170:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776965.981172:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776965.981175:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776965.981178:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776965.981181:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776965.981183:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776965.981185:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776965.981188:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981196:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981199:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981205:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981206:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776965.981209:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776965.981213:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776965.981215:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776965.981217:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313353733 00000100:00000001:0.0:1775776965.981219:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313354997 00000100:00000001:0.0:1775776965.981220:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313353733 00000100:00000001:0.0:1775776965.981221:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313354997 00000100:00000001:0.0:1775776965.981222:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776965.981224:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981225:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776965.981227:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776965/real 1775776965] req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776965.981231:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776965.981232:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776965.981234:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776965.981235:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776965.981236:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776965.981237:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776965.981239:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776965.981244:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981247:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776965.981248:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981249:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776965.981250:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776965.981251:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981255:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776965.981257:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981260:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776965.981261:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981262:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776965.981263:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981264:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776965.981265:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776965.981266:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776965.981268:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776965.981270:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776965.981272:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776965.981277:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099816960:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776965.981279:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776965.981281:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981285:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776965.981286:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776965.981287:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776965.981288:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776965.981289:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776965.981290:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817088:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776965.981292:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776965.981293:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776965.981294:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776965.981295:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981297:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d000. 02000000:00000001:0.0:1775776965.981298:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776965.981300:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776965.981301:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776965.981303:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776965.981305:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099817088, portal 10 00000100:00000001:0.0:1775776965.981306:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776965.981307:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288710656 : -114841420840960 : ffff978d65e7c000) 00000100:00000040:0.0:1775776965.981311:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981315:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776965.981316:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776965.981317:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776965.981319:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099817088, offset 0 00000400:00000200:0.0:1775776965.981321:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776965.981323:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.981325:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776965.981326:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776965.981329:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776965.981331:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776965.981334:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776965.981336:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4614:4614:256:4294967295] 192.168.201.139@tcp LPNI seq info [4614:4614:8:4294967295] 00000400:00000200:0.0:1775776965.981340:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.981345:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776965.981349:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776965.981352:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776965.981355:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776965.981358:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776965.981362:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776965.981364:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776965.981367:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776965.981369:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776965.981372:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776965.981375:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776965.981378:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776965.981379:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776965.981382:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776965.981385:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981390:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981391:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981395:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981396:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776965.981398:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776965.981400:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776965.981402:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776965.981403:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313353734 00000100:00000001:0.0:1775776965.981405:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313354997 00000100:00000001:0.0:1775776965.981406:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313353734 00000100:00000001:0.0:1775776965.981407:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313354997 00000100:00000001:0.0:1775776965.981409:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776965.981410:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981411:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776965.981413:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776965/real 1775776965] req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776965.981417:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776965.981418:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776965.981419:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776965.981420:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776965.981420:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776965.981421:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776965.981423:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776965.981427:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981431:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776965.981432:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981433:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776965.981433:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776965.981435:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981439:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776965.981440:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981444:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776965.981445:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981446:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776965.981446:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981448:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776965.981449:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776965.981450:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776965.981452:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776965.981453:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776965.981455:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776965.981459:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817088:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776965.981461:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776965.981463:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981466:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776965.981467:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776965.981468:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776965.981470:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776965.981471:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776965.981472:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817216:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776965.981474:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776965.981475:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776965.981476:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776965.981477:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981479:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775776965.981480:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776965.981482:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776965.981483:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776965.981485:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776965.981487:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099817216, portal 4 00000100:00000001:0.0:1775776965.981488:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776965.981489:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716032 : -114841420835584 : ffff978d65e7d500) 00000100:00000040:0.0:1775776965.981492:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981496:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776965.981497:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776965.981498:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776965.981500:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099817216, offset 0 00000400:00000200:0.0:1775776965.981502:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776965.981504:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.981506:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776965.981507:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776965.981509:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776965.981510:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776965.981512:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776965.981513:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4615:4615:256:4294967295] 192.168.201.139@tcp LPNI seq info [4615:4615:8:4294967295] 00000400:00000200:0.0:1775776965.981516:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.981519:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776965.981521:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776965.981523:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776965.981525:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776965.981527:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776965.981529:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776965.981531:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776965.981533:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776965.981534:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776965.981536:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776965.981538:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776965.981540:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776965.981541:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776965.981542:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776965.981544:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981547:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981549:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981557:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981558:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776965.981559:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776965.981561:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776965.981562:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776965.981563:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313353743 00000100:00000001:0.0:1775776965.981565:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313354997 00000100:00000001:0.0:1775776965.981566:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313353743 00000100:00000001:0.0:1775776965.981567:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313354997 00000100:00000001:0.0:1775776965.981568:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776965.981569:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981570:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776965.981572:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776965/real 1775776965] req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776965.981575:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776965.981576:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776965.981577:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776965.981578:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776965.981579:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776965.981580:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776965.981582:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776965.981586:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981589:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776965.981590:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981591:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776965.981591:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776965.981593:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981597:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776965.981599:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981602:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776965.981604:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981604:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776965.981605:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981606:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776965.981607:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776965.981608:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776965.981610:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776965.981611:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776965.981612:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776965.981616:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817216:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776965.981618:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776965.981620:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981623:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776965.981624:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776965.981625:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776965.981626:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776965.981627:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776965.981628:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817344:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776965.981630:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776965.981631:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776965.981632:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776965.981633:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981636:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775776965.981637:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776965.981639:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776965.981640:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776965.981642:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776965.981643:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099817344, portal 4 00000100:00000001:0.0:1775776965.981644:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776965.981645:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288723200 : -114841420828416 : ffff978d65e7f100) 00000100:00000040:0.0:1775776965.981649:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981652:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776965.981653:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776965.981655:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776965.981656:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099817344, offset 0 00000400:00000200:0.0:1775776965.981658:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776965.981659:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.981661:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776965.981662:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776965.981663:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776965.981665:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776965.981666:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776965.981669:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4616:4616:256:4294967295] 192.168.201.139@tcp LPNI seq info [4616:4616:8:4294967295] 00000400:00000200:0.0:1775776965.981672:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776965.981677:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776965.981681:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776965.981695:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776965.981698:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776965.981701:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776965.981705:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776965.981707:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776965.981710:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776965.981712:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776965.981714:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776965.981718:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776965.981720:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776965.981722:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776965.981724:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776965.981727:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981734:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981736:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776990 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981742:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981744:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776965.981746:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776965.981749:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776965.981751:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776965.981753:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313353734 00000100:00000001:0.0:1775776965.981754:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313354997 00000100:00000001:0.0:1775776965.981756:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313353734 00000100:00000001:0.0:1775776965.981757:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313354997 00000100:00000001:0.0:1775776965.981758:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776965.981759:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981760:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776965.981763:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776965/real 1775776965] req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776965.981767:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776965.981768:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776965.981769:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:0.0:1775776965.981770:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:0.0:1775776965.981771:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:0.0:1775776965.981772:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776965.981774:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776965.981793:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981797:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776965.981798:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981799:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776965.981800:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776965.981803:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981809:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776965.981813:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981818:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776965.981821:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981822:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776965.981823:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981825:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776965.981827:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776965.981829:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776965.981831:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776965.981834:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776965.981837:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776965.981845:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817344:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776965.981848:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981850:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981853:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e300 x1862033099816960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981859:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776965.981861:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981863:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775776965.981866:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776965.981867:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776965.981868:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775776965.981870:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776965.981871:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:0.0:1775776965.981874:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776965.981875:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776965.981876:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e300. 00000100:00000001:0.0:1775776965.981879:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776965.981880:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981882:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981885:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c000 x1862033099817088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981891:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776965.981892:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981894:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d000. 02000000:00000001:0.0:1775776965.981895:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776965.981896:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776965.981897:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775776965.981899:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776965.981900:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:0.0:1775776965.981902:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776965.981903:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776965.981904:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c000. 00000100:00000001:0.0:1775776965.981906:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776965.981908:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981909:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981912:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d500 x1862033099817216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981917:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776965.981919:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981920:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775776965.981921:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776965.981923:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776965.981924:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775776965.981925:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776965.981926:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:0.0:1775776965.981928:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776965.981929:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776965.981930:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d500. 00000100:00000001:0.0:1775776965.981932:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776965.981933:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981934:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776965.981938:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f100 x1862033099817344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776990 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776965.981944:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776965.981945:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776965.981946:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775776965.981949:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776965.981950:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776965.981951:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775776965.981953:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776965.981955:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:0.0:1775776965.981957:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776965.981958:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776965.981959:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f100. 00000100:00000001:0.0:1775776965.981961:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776965.981962:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981964:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776965.981967:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776965.981969:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776965.981971:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776965.981972:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776966.683931:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776966.683938:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776966.683940:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776966.683946:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776966.683950:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776966.683954:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776966.683957:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776966.683959:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776966.683962:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776966.747911:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776966.747917:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776966.747922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776966.747923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776966.747927:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776966.747928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776966.747932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776966.747933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776966.748034:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776966.748038:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776966.748041:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776966.875937:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776966.875947:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776966.875949:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776966.875957:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776966.875962:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776966.875967:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776966.875970:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776966.875974:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776966.875978:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776967.259936:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313354997, at 4313355008 00000100:00000010:1.0:1775776967.259953:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237900. 00000100:00000040:0.0:1775776967.259959:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313354997, at 4313355008 00000100:00000010:0.0:1775776967.259969:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775776967.707901:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776967.707909:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776967.707911:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776967.707916:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776967.707919:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776967.707923:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776967.707925:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776967.707927:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776967.707930:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776967.772356:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776967.772363:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776967.772370:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776967.772371:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776967.772376:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776967.772377:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776967.772381:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776967.772383:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776967.773105:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776967.773108:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776967.773111:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776967.899872:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776967.899880:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776967.899882:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776967.899888:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776967.899893:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776967.899897:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776967.899900:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776967.899903:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776967.899906:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776968.732106:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776968.732114:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776968.732116:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776968.732122:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776968.732126:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776968.732130:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776968.732132:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776968.732135:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776968.732137:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776968.795900:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776968.795906:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776968.795908:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776968.795911:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776968.795914:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776968.795916:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776968.795917:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776968.795919:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776968.795922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776968.795924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776968.795927:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776968.795928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776968.795932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776968.795933:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776968.795987:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776968.795990:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776968.795993:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776968.924009:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776968.924016:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776968.924018:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776968.924023:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776968.924027:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776968.924031:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776968.924033:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776968.924035:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776968.924038:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776969.756065:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776969.756072:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776969.756073:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776969.756079:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776969.756082:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776969.756085:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776969.756088:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776969.756090:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776969.756092:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776969.819879:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776969.819884:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776969.819888:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776969.819889:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776969.819891:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776969.819892:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776969.819895:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776969.819896:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776969.819974:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776969.819976:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776969.819979:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776969.947959:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776969.947967:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776969.947968:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776969.947974:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776969.947977:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776969.947981:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776969.947984:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776969.947986:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776969.947989:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776970.780007:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776970.780015:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776970.780017:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776970.780023:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776970.780027:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776970.780031:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776970.780033:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776970.780036:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776970.780039:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776970.843924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776970.843932:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776970.843934:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776970.843938:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843941:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776970.843943:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843945:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843948:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843951:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776970.843953:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776970.843954:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776970.843956:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843957:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776970.843958:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843960:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843962:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843967:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776970.843969:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776970.843970:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776970.843971:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843973:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776970.843974:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843975:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776970.843977:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776970.844261:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776970.844264:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776970.844267:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776970.971904:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776970.971911:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776970.971913:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776970.971918:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776970.971922:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776970.971926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776970.971929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776970.971931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776970.971934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776971.099928:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776971.099939:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776971.099942:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776971.099946:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776971.099950:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776971.099957:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18686 00000400:00000200:0.0:1775776971.099967:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776971.099974:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776971.099977:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776971.099980:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.099984:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.099986:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.099989:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776971.099992:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776971.099994:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776971.099997:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776971.099999:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776971.100002:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776971.100004:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776971.100006:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100008:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.100010:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100011:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776971.100014:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776971.100017:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100019:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776971.100022:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776971.100025:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776971.100027:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100030:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776971.100032:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776971.100042:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df100. 00000020:00000040:0.0:1775776971.100047:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776971.100051:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776971.100052:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100056:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776971.100058:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776971.100062:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776971.100069:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d400. 00000100:00000001:0.0:1775776971.100075:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776971.100093:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776971.100102:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776971.100107:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776971.100111:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072f03875] to pc [ptlrpcd_rcv+-1] req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776971.100141:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776971.100145:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776971.100148:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776971.100149:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776971.100151:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776971.100153:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776971.100155:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18686 00000400:00000200:0.0:1775776971.100159:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776971.100162:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776971.100164:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776971.100167:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100169:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.100170:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100172:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776971.100174:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776971.100175:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776971.100188:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776971.100190:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776971.100192:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776971.100194:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776971.100197:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100199:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.100200:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100202:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776971.100204:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776971.100206:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100209:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776971.100211:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776971.100216:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776971.100218:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100220:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776971.100224:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776971.100227:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100230:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776971.100233:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df800. 00000020:00000040:0.0:1775776971.100236:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776971.100238:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776971.100240:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100241:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776971.100243:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776971.100245:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776971.100248:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:0.0:1775776971.100251:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776971.100256:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776971.100264:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776971.100266:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776971.100270:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000433eaeb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776971.100283:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776971.100286:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776971.100290:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776971.100291:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776971.100292:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776971.100295:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776971.100298:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18686 00000400:00000200:0.0:1775776971.100301:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776971.100304:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776971.100306:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776971.100309:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100311:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.100313:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100315:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776971.100318:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776971.100319:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776971.100320:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776971.100322:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776971.100324:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776971.100327:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776971.100330:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100331:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.100333:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100335:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776971.100337:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776971.100340:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100342:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776971.100344:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776971.100347:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776971.100349:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100351:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776971.100353:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776971.100356:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100357:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776971.100361:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd500. 00000020:00000040:0.0:1775776971.100363:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776971.100366:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776971.100367:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100368:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776971.100370:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776971.100371:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776971.100375:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776971.100378:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776971.100382:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776971.100389:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776971.100391:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776971.100395:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a01aaede] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776971.100402:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776971.100404:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776971.100407:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776971.100408:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776971.100409:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776971.100411:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776971.100414:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18686 00000400:00000200:0.0:1775776971.100417:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776971.100419:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776971.100422:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776971.100424:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100426:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.100428:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100429:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776971.100432:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776971.100433:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776971.100434:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776971.100436:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776971.100438:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776971.100440:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776971.100443:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100445:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.100446:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100448:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776971.100450:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776971.100453:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776971.100455:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776971.100457:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776971.100460:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776971.100461:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100463:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776971.100465:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776971.100467:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100469:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776971.100481:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dea00. 00000020:00000040:0.0:1775776971.100484:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776971.100487:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776971.100488:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.100489:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776971.100491:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776971.100492:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776971.100496:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776971.100500:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776971.100504:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776971.100510:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776971.100512:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776971.100515:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000090d94a6] to pc [ptlrpcd_rcv+-1] req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776971.100522:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776971.100525:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776971.100528:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776971.100529:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776971.100531:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776971.100533:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776971.100535:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18686 00000100:00000001:2.0:1775776971.100660:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776971.100664:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776971.100666:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776971.100672:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.100691:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776971.100693:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.100696:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776971.100699:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776971.100702:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776971.100705:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099817472:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776971.100709:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776971.100713:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776971.100716:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.100719:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776971.100724:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776971.100726:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776971.100731:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776971.100735:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776971.100741:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776971.100749:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099817472, portal 25 00000100:00000001:2.0:1775776971.100752:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776971.100753:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741759744 : -114841967791872 : ffff978d454df100) 00000100:00000040:2.0:1775776971.100759:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.100766:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776971.100892:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776971.100896:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776971.100900:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099817472, offset 0 00000400:00000200:2.0:1775776971.100907:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776971.100916:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.100923:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776971.100928:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776971.100956:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776971.100960:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776971.100963:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776971.100967:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4617:4617:256:4294967295] 192.168.201.139@tcp LPNI seq info [4617:4617:8:4294967295] 00000400:00000200:2.0:1775776971.100972:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.100983:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776971.100989:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.100994:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776971.101001:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776971.101005:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776971.101010:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776971.101014:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776971.101018:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.101020:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776971.101027:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776971.101032:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776971.101037:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776971.101040:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776971.101044:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776971.101048:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101057:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.101060:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101068:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.101069:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776971.101076:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776971.101080:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776971.101083:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313360118 00000100:00000010:2.0:1775776971.101088:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc00. 00000100:00000001:2.0:1775776971.101094:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776971.101101:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313360118 00000100:00000010:2.0:1775776971.101106:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bcc0. 00000100:00000001:2.0:1775776971.101109:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776971.101112:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776971.101115:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.101117:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776971.101121:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776971/real 1775776971] req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776971.101131:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776971.101133:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776971.101134:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776971.101137:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776971.101138:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776971.101141:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776971.101144:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776971.101153:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101160:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776971.101162:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.101164:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.101166:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776971.101170:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101177:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776971.101182:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101190:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776971.101192:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.101194:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.101195:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.101197:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776971.101199:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776971.101202:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776971.101206:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776971.101208:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776971.101212:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776971.101221:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099817472:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776971.101225:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776971.101228:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101235:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776971.101236:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.101239:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776971.101240:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776971.101242:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776971.101245:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817600:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776971.101248:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776971.101250:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776971.101252:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.101254:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776971.101257:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776971.101258:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776971.101261:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776971.101265:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776971.101268:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776971.101271:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099817600, portal 10 00000100:00000001:2.0:1775776971.101273:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776971.101274:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741761536 : -114841967790080 : ffff978d454df800) 00000100:00000040:2.0:1775776971.101281:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101287:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776971.101290:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776971.101292:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776971.101295:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099817600, offset 0 00000400:00000200:2.0:1775776971.101299:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776971.101303:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.101307:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776971.101310:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776971.101312:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776971.101314:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776971.101317:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776971.101319:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4618:4618:256:4294967295] 192.168.201.139@tcp LPNI seq info [4618:4618:8:4294967295] 00000400:00000200:2.0:1775776971.101323:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.101329:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776971.101334:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.101337:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776971.101341:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776971.101343:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776971.101348:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776971.101350:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776971.101355:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.101356:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776971.101359:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776971.101363:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776971.101365:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776971.101367:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776971.101370:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776971.101374:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101381:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.101384:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101390:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.101391:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776971.101394:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776971.101398:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776971.101401:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776971.101403:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313358853 00000100:00000001:2.0:1775776971.101405:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313360118 00000100:00000001:2.0:1775776971.101407:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313358853 00000100:00000001:2.0:1775776971.101409:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313360118 00000100:00000001:2.0:1775776971.101411:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776971.101413:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.101414:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776971.101418:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776971/real 1775776971] req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776971.101425:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776971.101426:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776971.101428:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776971.101430:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776971.101432:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776971.101434:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776971.101436:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776971.101445:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101457:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776971.101459:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.101460:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.101462:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776971.101465:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101472:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776971.101476:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101483:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776971.101485:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.101487:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.101488:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.101490:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776971.101492:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776971.101494:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776971.101497:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776971.101499:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776971.101503:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776971.101511:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817600:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776971.101514:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776971.101517:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101524:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776971.101525:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.101527:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776971.101529:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776971.101531:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776971.101534:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817728:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776971.101537:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776971.101539:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776971.101541:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.101542:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776971.101546:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776971.101548:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776971.101551:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776971.101554:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776971.101557:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776971.101560:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099817728, portal 10 00000100:00000001:2.0:1775776971.101562:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776971.101563:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741752576 : -114841967799040 : ffff978d454dd500) 00000100:00000040:2.0:1775776971.101569:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101575:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776971.101577:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776971.101580:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776971.101582:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099817728, offset 0 00000400:00000200:2.0:1775776971.101585:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776971.101589:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.101593:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776971.101630:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776971.101633:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776971.101635:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776971.101638:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776971.101641:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4619:4619:256:4294967295] 192.168.201.139@tcp LPNI seq info [4619:4619:8:4294967295] 00000400:00000200:2.0:1775776971.101645:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.101651:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776971.101656:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.101659:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776971.101663:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776971.101665:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776971.101669:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776971.101672:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776971.101675:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.101676:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776971.101679:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776971.101683:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776971.101686:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776971.101687:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776971.101690:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776971.101694:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101701:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.101704:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.101710:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.101712:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776971.101715:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776971.101718:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776971.101721:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776971.101723:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313358854 00000100:00000001:2.0:1775776971.101725:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313360118 00000100:00000001:2.0:1775776971.101727:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313358854 00000100:00000001:2.0:1775776971.101729:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313360118 00000100:00000001:2.0:1775776971.101731:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776971.101733:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.101735:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776971.101738:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776971/real 1775776971] req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776971.101746:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776971.101747:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776971.101748:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776971.101751:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776971.101752:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776971.101754:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776971.101757:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776971.101764:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776971.102033:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776971.102037:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776971.102040:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776971.102043:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.102046:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.102048:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.102050:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776971.102053:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776971.102071:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776971.102074:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776971.102076:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776971.102078:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776971.102080:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776971.102084:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.102085:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776971.102088:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776971.102089:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776971.102092:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776971.102095:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776971.102097:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776971.102099:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776971.102103:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776971.102105:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.102107:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776971.102110:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776971.102126:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.102128:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776971.102133:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ded80. 00000020:00000040:0.0:1775776971.102135:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776971.102138:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776971.102139:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776971.102142:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776971.102144:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776971.102146:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776971.102152:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776971.102156:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776971.102162:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776971.102169:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776971.102172:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776971.102187:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d96bdcf3] to pc [ptlrpcd_rcv+-1] req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776971.102196:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776971.102201:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18696) 00000100:00000001:2.0:1775776971.102864:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776971.102866:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.102869:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.102870:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776971.102876:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.102882:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776971.102886:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.102892:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776971.102894:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.102896:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.102897:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.102899:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776971.102901:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776971.102904:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776971.102908:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776971.102910:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776971.102915:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776971.102922:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817728:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776971.102926:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776971.102930:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.102936:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776971.102938:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.102940:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776971.102942:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776971.102943:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776971.102946:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817856:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776971.102949:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776971.102951:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776971.102953:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.102955:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776971.102959:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776971.102962:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776971.102966:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776971.102970:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776971.102974:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776971.102979:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099817856, portal 4 00000100:00000001:2.0:1775776971.102980:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776971.102982:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757952 : -114841967793664 : ffff978d454dea00) 00000100:00000040:2.0:1775776971.102988:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.102994:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776971.102997:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776971.102999:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776971.103002:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099817856, offset 0 00000400:00000200:2.0:1775776971.103008:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776971.103012:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.103016:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776971.103019:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776971.103021:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776971.103024:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776971.103026:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776971.103029:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4620:4620:256:4294967295] 192.168.201.139@tcp LPNI seq info [4620:4620:8:4294967295] 00000400:00000200:2.0:1775776971.103033:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.103039:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776971.103043:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.103046:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776971.103051:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776971.103053:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776971.103057:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776971.103060:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776971.103064:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.103065:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776971.103069:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776971.103072:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776971.103075:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776971.103077:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776971.103079:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776971.103083:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103090:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.103093:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103099:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.103101:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776971.103104:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776971.103108:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776971.103111:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776971.103113:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313358865 00000100:00000001:2.0:1775776971.103115:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313360118 00000100:00000001:2.0:1775776971.103117:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313358865 00000100:00000001:2.0:1775776971.103119:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313360118 00000100:00000001:2.0:1775776971.103121:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776971.103123:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.103125:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776971.103128:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776971/real 1775776971] req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776971.103136:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776971.103137:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776971.103139:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776971.103141:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776971.103142:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776971.103144:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776971.103147:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776971.103154:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103160:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776971.103162:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103164:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.103165:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776971.103168:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103175:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776971.103178:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103184:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776971.103271:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103273:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.103274:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103276:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776971.103277:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776971.103280:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776971.103283:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776971.103285:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776971.103289:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776971.103297:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817856:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776971.103301:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103303:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.103306:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df100 x1862033099817472/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103313:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776971.103315:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776971.103318:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776971.103321:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776971.103322:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776971.103324:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776971.103326:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776971.103329:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d400. 02000000:00000001:2.0:1775776971.103332:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776971.103334:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776971.103336:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df100. 00000100:00000001:2.0:1775776971.103339:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776971.103340:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103341:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.103345:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df800 x1862033099817600/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103356:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776971.103357:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776971.103359:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776971.103361:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776971.103362:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776971.103363:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776971.103365:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776971.103367:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:2.0:1775776971.103368:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776971.103369:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776971.103370:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df800. 00000100:00000001:2.0:1775776971.103372:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776971.103373:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103374:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.103377:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd500 x1862033099817728/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103383:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776971.103384:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776971.103385:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776971.103386:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776971.103387:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776971.103388:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776971.103390:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776971.103391:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775776971.103394:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776971.103395:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776971.103396:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd500. 00000100:00000001:2.0:1775776971.103397:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776971.103399:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103400:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.103402:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dea00 x1862033099817856/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103408:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776971.103409:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776971.103411:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776971.103412:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776971.103413:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776971.103415:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776971.103416:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776971.103418:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:2.0:1775776971.103420:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776971.103421:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776971.103422:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dea00. 00000100:00000001:2.0:1775776971.103424:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776971.103425:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103428:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103435:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776971.103437:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.103439:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776971.103441:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776971.103442:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776971.103445:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103452:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776971.103453:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.103456:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776971.103457:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776971.103459:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776971.103462:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817984:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776971.103465:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776971.103467:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776971.103468:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776971.103470:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776971.103473:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776971.103474:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776971.103477:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776971.103479:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776971.103482:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776971.103485:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099817984, portal 4 00000100:00000001:2.0:1775776971.103487:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776971.103488:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741758848 : -114841967792768 : ffff978d454ded80) 00000100:00000040:2.0:1775776971.103493:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103498:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776971.103500:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776971.103502:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776971.103504:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099817984, offset 0 00000400:00000200:2.0:1775776971.103507:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776971.103510:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.103513:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776971.103516:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776971.103518:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776971.103520:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776971.103522:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776971.103524:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4621:4621:256:4294967295] 192.168.201.139@tcp LPNI seq info [4621:4621:8:4294967295] 00000400:00000200:2.0:1775776971.103528:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776971.103535:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776971.103539:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.103541:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776971.103545:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776971.103547:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1775776971.103550:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776971.103553:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1775776971.103608:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776971.103611:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776971.103614:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776971.103617:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776971.103621:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776971.103622:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776971.103625:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776971.103628:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103756:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.103760:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775776996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103766:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.103859:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776971.103863:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776971.103867:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776971.103870:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776971.103872:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313358857 00000100:00000001:2.0:1775776971.103874:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313360118 00000100:00000001:2.0:1775776971.103876:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313358857 00000100:00000001:2.0:1775776971.103877:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313360118 00000100:00000001:2.0:1775776971.103879:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776971.103881:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.103883:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776971.103886:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776971/real 1775776971] req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776971.103893:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776971.103894:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776971.103896:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776971.103898:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776971.103899:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776971.103901:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776971.103904:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776971.103911:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103917:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776971.103918:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103920:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.103921:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776971.103924:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103929:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776971.103932:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103939:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776971.103941:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103942:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776971.103943:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103945:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776971.103947:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776971.103949:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776971.103952:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776971.103954:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776971.103958:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776971.103965:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099817984:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776971.103969:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.103971:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776971.103973:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ded80 x1862033099817984/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775776996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776971.103979:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776971.103981:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776971.103983:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776971.103985:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776971.103986:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776971.103988:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776971.103990:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776971.103991:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:2.0:1775776971.103993:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776971.103994:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776971.103995:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ded80. 00000100:00000001:2.0:1775776971.103998:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776971.103999:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.104001:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776971.104004:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776971.104006:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776971.104007:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776971.104008:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776971.803893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776971.803900:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776971.803902:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776971.803906:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776971.803910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776971.803914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776971.803917:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776971.803919:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776971.803922:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776971.867917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776971.867924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776971.867930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776971.867931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776971.867935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776971.867936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776971.867939:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776971.867940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776971.868007:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776971.868011:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776971.868015:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776971.995902:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776971.995911:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776971.995913:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776971.995918:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776971.995922:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776971.995926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776971.995928:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776971.995931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776971.995934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776972.380011:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313360118, at 4313360128 00000100:00000010:0.0:1775776972.380023:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc00. 00000100:00000040:1.0:1775776972.380100:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313360118, at 4313360128 00000100:00000010:1.0:1775776972.380106:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bcc0. 00000100:00000001:3.0:1775776972.827872:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776972.827879:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776972.827884:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776972.827892:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776972.827895:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776972.827902:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776972.827904:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776972.827907:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776972.827909:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776972.891878:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776972.891883:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776972.891887:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776972.891888:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776972.891890:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776972.891891:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776972.891893:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776972.891893:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776972.891985:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776972.891989:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776972.891992:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776973.019898:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776973.019906:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776973.019908:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776973.019914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776973.019918:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776973.019921:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776973.019924:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776973.019926:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776973.019929:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776973.851866:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776973.851873:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776973.851875:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776973.851881:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776973.851884:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776973.851888:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776973.851891:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776973.851893:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776973.851895:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776973.915942:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776973.915948:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776973.915953:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776973.915954:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776973.915958:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776973.915959:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776973.915962:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776973.915963:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776973.916047:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776973.916051:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776973.916054:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776974.046276:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776974.046283:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776974.046285:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776974.046290:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776974.046293:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776974.046297:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776974.046299:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776974.046301:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776974.046304:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776974.875897:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776974.875904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776974.875906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776974.875911:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776974.875915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776974.875918:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776974.875921:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776974.875923:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776974.875926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776974.939895:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776974.939901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776974.939906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776974.939907:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776974.939910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776974.939911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776974.939914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776974.939915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776974.940278:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776974.940283:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776974.940285:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776975.067879:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776975.067886:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776975.067888:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776975.067893:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776975.067896:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776975.067899:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776975.067901:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776975.067903:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776975.067906:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776975.899845:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776975.899851:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776975.899853:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776975.899858:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776975.899862:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776975.899865:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776975.899868:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776975.899870:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776975.899873:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776975.963866:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776975.963872:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776975.963877:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776975.963878:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776975.963882:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776975.963883:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776975.963886:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776975.963887:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776975.963962:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776975.963965:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776975.963968:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776976.093638:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776976.093646:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776976.093647:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776976.093653:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776976.093656:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776976.093660:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776976.093663:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776976.093665:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776976.093667:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776976.220041:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776976.220052:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776976.220055:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776976.220058:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776976.220062:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776976.220069:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18691 00000400:00000200:2.0:1775776976.220078:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776976.220084:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776976.220087:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776976.220090:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220094:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.220096:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220099:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776976.220102:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776976.220104:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775776976.220107:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776976.220109:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775776976.220111:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775776976.220113:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775776976.220117:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220119:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.220121:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220123:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776976.220125:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775776976.220128:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220130:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776976.220133:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776976.220136:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776976.220139:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220140:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776976.220142:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776976.220150:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f480. 00000020:00000040:2.0:1775776976.220153:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775776976.220156:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776976.220158:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220161:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776976.220163:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776976.220167:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776976.220173:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775776976.220179:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776976.220187:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776976.220197:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776976.220201:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776976.220206:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002c91b8b4] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.220338:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776976.220342:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776976.220345:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776976.220350:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.220358:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776976.220360:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.220363:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776976.220366:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776976.220369:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776976.220372:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099818112:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775776976.220376:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776976.220379:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776976.220382:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.220384:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776976.220389:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775776976.220391:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776976.220396:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776976.220400:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306688. 00000400:00000010:0.0:1775776976.220406:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8aa0. 00000100:00000200:0.0:1775776976.220413:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099818112, portal 25 00000100:00000001:0.0:1775776976.220415:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776976.220416:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724096 : -114841420827520 : ffff978d65e7f480) 00000100:00000040:0.0:1775776976.220421:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.220429:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776976.220431:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776976.220433:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775776976.220436:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099818112, offset 0 00000400:00000200:0.0:1775776976.220442:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776976.220449:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.220455:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776976.220459:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776976.220463:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776976.220467:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776976.220469:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776976.220473:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4622:4622:256:4294967295] 192.168.201.139@tcp LPNI seq info [4622:4622:8:4294967295] 00000400:00000200:0.0:1775776976.220477:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.220486:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776976.220492:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776976.220497:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775776976.220503:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:2.0:1775776976.220524:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776976.220527:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776976.220531:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776976.220532:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776976.220534:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776976.220537:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776976.220539:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18691 00000400:00000200:2.0:1775776976.220543:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776976.220546:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776976.220548:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776976.220550:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220552:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.220554:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220556:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776976.220559:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776976.220560:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775776976.220562:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776976.220564:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775776976.220566:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775776976.220568:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775776976.220571:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220572:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.220574:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220576:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776976.220578:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775776976.220580:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220582:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776976.220583:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776976.220588:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776976.220590:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220592:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776976.220595:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776976.220597:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220599:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776976.220603:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d880. 00000020:00000040:2.0:1775776976.220605:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776976.220608:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776976.220609:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776976.220628:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776976.220646:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776976.220650:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776976.220652:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313365237 00000100:00000010:0.0:1775776976.220656:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775776976.220662:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776976.220668:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313365237 00000100:00000010:0.0:1775776976.220671:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237a20. 00000100:00000001:0.0:1775776976.220674:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776976.220677:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776976.220679:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.220681:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.220684:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776976.220690:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776976.220692:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776976.220694:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776976.220695:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776976.220697:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.220699:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220730:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776976.220732:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776976.220734:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776976.220738:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504c00. 00000100:00000001:2.0:1775776976.220742:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776976.220746:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776976.220753:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776976.220756:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776976.220759:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f88f35f] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776976.220774:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776976.220777:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776976.220802:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776976.220803:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776976.220805:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776976.220807:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776976.220810:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18691 00000400:00000200:2.0:1775776976.220814:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776976.220817:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776976.220819:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776976.220822:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220823:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.220825:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220827:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776976.220829:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776976.220831:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775776976.220833:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776976.220835:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775776976.220837:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775776976.220839:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775776976.220841:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220843:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.220845:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220846:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776976.220848:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775776976.220851:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220853:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776976.220855:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776976.220857:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776976.220859:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220860:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776976.220862:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776976.220864:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220866:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776976.220869:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7dc00. 00000020:00000040:2.0:1775776976.220871:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775776976.220874:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776976.220875:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220877:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776976.220878:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776976.220880:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776976.220883:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507400. 00000100:00000001:2.0:1775776976.220885:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776976.220889:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776976.220896:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776976.220898:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776976.220902:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f1188f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776976.220909:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776976.220912:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776976.220915:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776976.220916:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776976.220918:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776976.220920:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776976.220922:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18691 00000400:00000200:2.0:1775776976.220925:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776976.220927:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776976.220929:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776976.220932:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220933:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.220935:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220937:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776976.220939:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776976.220940:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775776976.220942:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776976.220943:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775776976.220945:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775776976.220947:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775776976.220949:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220951:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.220953:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220955:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776976.220957:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775776976.220959:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776976.220961:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776976.220962:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776976.220965:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776976.220966:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220968:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776976.220970:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776976.220972:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220974:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776976.220977:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7df80. 00000020:00000040:2.0:1775776976.220978:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775776976.220980:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776976.220981:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.220983:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776976.220984:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776976.220985:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776976.220988:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504400. 00000100:00000001:2.0:1775776976.220991:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776976.220994:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776976.221001:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776976.221003:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776976.221006:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009deec8f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776976.221013:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776976.221015:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776976.221017:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775776976.221019:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776976.221020:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776976.221022:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776976.221024:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18691 00000400:00000200:2.0:1775776976.221027:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776976.221029:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776976.221031:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776976.221034:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.221036:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.221038:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.221040:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775776976.221042:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776976.221043:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775776976.221044:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776976.221045:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775776976.221047:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775776976.221049:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775776976.221051:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.221053:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776976.221054:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776976.221056:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775776976.221059:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775776976.221061:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776976.221063:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776976.221064:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776976.221066:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776976.221067:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.221069:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776976.221071:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775776976.221073:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.221074:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776976.221076:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c380. 00000020:00000040:2.0:1775776976.221078:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775776976.221080:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776976.221082:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776976.221083:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776976.221084:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776976.221086:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776976.221088:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775776976.221091:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776976.221095:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776976.221102:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776976.221104:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776976.221107:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f231afe] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776976.221115:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776976.221119:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18701) 00000100:00000001:0.0:1775776976.221164:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776976.221166:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776976.221168:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776976.221172:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.221178:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776976.221180:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.221182:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776976.221184:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776976.221186:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776976.221189:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818240:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776976.221192:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776976.221194:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776976.221195:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.221196:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776976.221200:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775776976.221202:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776976.221205:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776976.221208:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513067e8. 00000400:00000010:0.0:1775776976.221211:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b85d8. 00000100:00000200:0.0:1775776976.221215:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099818240, portal 10 00000100:00000001:0.0:1775776976.221216:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776976.221217:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716928 : -114841420834688 : ffff978d65e7d880) 00000100:00000040:0.0:1775776976.221223:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.221230:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776976.221232:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776976.221234:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8b28. 00000100:00000200:0.0:1775776976.221236:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099818240, offset 0 00000400:00000200:0.0:1775776976.221240:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776976.221244:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.221247:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776976.221250:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776976.221252:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776976.221255:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776976.221257:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776976.221261:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4623:4623:255:4294967295] 192.168.201.139@tcp LPNI seq info [4623:4623:7:4294967295] 00000400:00000200:0.0:1775776976.221265:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.221270:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776976.221275:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776976.221278:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bd00. 00000800:00000200:0.0:1775776976.221282:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776976.221286:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776976.221289:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776976.221292:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776976.221295:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776976.221296:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313363973 00000100:00000001:0.0:1775776976.221299:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313365237 00000100:00000001:0.0:1775776976.221301:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313363973 00000100:00000001:0.0:1775776976.221303:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313365237 00000100:00000001:0.0:1775776976.221305:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776976.221307:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.221309:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776976.221311:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.221318:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776976.221319:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.221321:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776976.221322:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776976.221324:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776976.221326:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818368:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776976.221329:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776976.221331:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776976.221332:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.221334:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776976.221336:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775776976.221338:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776976.221340:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776976.221342:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069f8. 00000400:00000010:0.0:1775776976.221345:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8880. 00000100:00000200:0.0:1775776976.221348:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099818368, portal 10 00000100:00000001:0.0:1775776976.221349:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776976.221350:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288717824 : -114841420833792 : ffff978d65e7dc00) 00000100:00000040:0.0:1775776976.221354:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.221361:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776976.221363:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776976.221365:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775776976.221367:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099818368, offset 0 00000400:00000200:0.0:1775776976.221370:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776976.221373:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.221376:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776976.221378:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776976.221380:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776976.221382:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776976.221384:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776976.221387:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4624:4624:254:4294967295] 192.168.201.139@tcp LPNI seq info [4624:4624:6:4294967295] 00000400:00000200:0.0:1775776976.221391:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.221397:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776976.221400:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776976.221403:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775776976.221406:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776976.221409:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776976.221412:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776976.221414:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775776976.221416:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776976.221418:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313363974 00000100:00000001:0.0:1775776976.221420:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313365237 00000100:00000001:0.0:1775776976.221422:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313363974 00000100:00000001:0.0:1775776976.221424:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313365237 00000100:00000001:0.0:1775776976.221426:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776976.221428:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.221430:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776976.221433:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.221439:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776976.221441:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.221443:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776976.221444:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776976.221446:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776976.221449:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818496:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776976.221451:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776976.221453:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776976.221454:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.221455:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776976.221457:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775776976.221459:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776976.221462:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776976.221464:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306c60. 00000400:00000010:0.0:1775776976.221467:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8908. 00000100:00000200:0.0:1775776976.221469:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099818496, portal 4 00000100:00000001:0.0:1775776976.221471:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776976.221472:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288718720 : -114841420832896 : ffff978d65e7df80) 00000100:00000040:0.0:1775776976.221476:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.221482:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776976.221484:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776976.221486:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775776976.221488:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099818496, offset 0 00000400:00000200:0.0:1775776976.221493:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776976.221496:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.221498:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776976.221501:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776976.221503:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776976.226278:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776976.226282:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776976.226287:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4625:4625:253:4294967295] 192.168.201.139@tcp LPNI seq info [4625:4625:5:4294967295] 00000400:00000200:0.0:1775776976.226295:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.226305:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776976.226312:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776976.226322:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b300. 00000800:00000200:0.0:1775776976.226329:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776976.226333:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776976.226339:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776976.226344:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775776976.226348:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776976.226350:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313363988 00000100:00000001:0.0:1775776976.226354:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313365237 00000100:00000001:0.0:1775776976.226356:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313363988 00000100:00000001:0.0:1775776976.226358:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313365237 00000100:00000001:0.0:1775776976.226360:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776976.226364:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.226368:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776976.226374:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.226384:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776976.226385:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.226389:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776976.226392:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776976.226395:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776976.226398:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818624:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775776976.226402:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776976.226405:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776976.226408:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776976.226411:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776976.226418:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775776976.226419:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776976.226424:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776976.226428:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306e70. 00000400:00000010:0.0:1775776976.226435:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b84c8. 00000100:00000200:0.0:1775776976.226440:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099818624, portal 4 00000100:00000001:0.0:1775776976.226442:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776976.226444:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288711552 : -114841420840064 : ffff978d65e7c380) 00000100:00000040:0.0:1775776976.226450:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776976.226457:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776976.226459:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776976.226461:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775776976.226464:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099818624, offset 0 00000400:00000200:0.0:1775776976.226470:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776976.226476:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.226482:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776976.227341:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776976.227346:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776976.227348:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776976.227351:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776976.227355:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4626:4626:252:4294967295] 192.168.201.139@tcp LPNI seq info [4626:4626:4:4294967295] 00000400:00000200:0.0:1775776976.227360:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776976.227367:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776976.227373:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776976.227378:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775776976.227382:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776976.227386:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775776976.227389:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776976.227393:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775776976.227396:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776976.227398:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313363980 00000100:00000001:0.0:1775776976.227401:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313365237 00000100:00000001:0.0:1775776976.227403:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313363980 00000100:00000001:0.0:1775776976.227404:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313365237 00000100:00000001:0.0:1775776976.227406:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776976.227409:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.227413:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.227417:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776976.227423:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776976.227425:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776976.227428:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776976.227429:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776976.227432:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776976.227434:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776976.923893:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776976.923899:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776976.923901:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776976.923907:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776976.923910:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776976.923914:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776976.923916:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776976.923918:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776976.923921:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776976.988118:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776976.988124:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776976.988130:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776976.988132:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776976.988135:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776976.988136:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776976.988139:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776976.988140:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776976.988218:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776976.988222:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776976.988224:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776977.115862:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776977.115869:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776977.115871:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776977.115876:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776977.115879:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776977.115883:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776977.115885:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776977.115888:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776977.115890:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776977.499997:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313365237, at 4313365248 00000100:00000010:0.0:1775776977.500021:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775776977.501582:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313365237, at 4313365249 00000100:00000010:1.0:1775776977.501600:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237a20. 00000100:00000001:3.0:1775776977.948011:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776977.948022:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776977.948025:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776977.948043:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776977.948049:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776977.948056:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776977.948059:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776977.948070:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776977.948075:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776978.012825:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776978.012832:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776978.012840:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776978.012842:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776978.012846:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776978.012847:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776978.012851:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776978.012853:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776978.012932:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776978.012938:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776978.012943:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776978.141252:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776978.141260:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776978.141262:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776978.141268:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776978.141273:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776978.141278:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776978.141281:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776978.141285:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776978.141289:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776978.971970:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776978.971978:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776978.971980:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776978.971987:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776978.971992:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776978.971998:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776978.972001:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776978.972004:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776978.972007:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776979.035969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776979.035978:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776979.035981:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776979.035985:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776979.035991:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776979.035994:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776979.035996:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776979.035999:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776979.036004:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776979.036005:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776979.036011:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776979.036012:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776979.036016:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776979.036017:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776979.037724:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776979.037730:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776979.037735:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776979.163924:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776979.163933:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776979.163935:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776979.163942:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776979.163948:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776979.163953:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776979.163957:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776979.163960:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776979.163965:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1775776979.293058:0:301877:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:0.0:1775776979.293144:0:301877:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775776979.293160:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776979.293165:0:301877:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:0.0:1775776979.293172:0:301877:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776979.293178:0:301877:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775776979.293186:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776979.293192:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:0.0:1775776979.293202:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776979.293208:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776979.293220:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775776979.295825:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775776979.295843:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776979.295855:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.295882:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776979.295886:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.295892:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776979.295893:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776979.295908:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776979.295911:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bd00. 00000400:00000200:0.0:1775776979.295916:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776979.295920:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776979.295925:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8b28 00000400:00000010:0.0:1775776979.295927:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8b28. 00000100:00000001:0.0:1775776979.295930:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776979.295935:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.295942:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776979.295945:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.295951:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776979.295952:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776979.295957:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776979.295959:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:0.0:1775776979.295961:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776979.295964:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776979.295967:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775776979.295969:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775776979.295971:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776979.295975:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.295981:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776979.295984:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.295989:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776979.295991:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776979.295996:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776979.295998:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b300. 00000400:00000200:0.0:1775776979.296001:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776979.296004:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776979.296006:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775776979.296008:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775776979.296010:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776979.296015:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.296021:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776979.296023:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.296027:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776979.296028:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776979.296031:0:301877:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:0.0:1775776979.296032:0:301877:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775776979.296034:0:301877:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776979.296035:0:301877:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776979.296037:0:301877:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775776979.296039:0:301877:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775776979.296040:0:301877:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776979.296043:0:301877:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.296047:0:301877:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776979.296049:0:301877:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776979.296052:0:301877:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776979.296053:0:301877:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776979.296606:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776979.296619:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776979.296622:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776979.296735:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776976/real 1775776979] req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776979.296751:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776979.296754:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776979.296758:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513067e8. 00000400:00000200:2.0:1775776979.296764:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b85d8 00000400:00000010:2.0:1775776979.296766:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b85d8. 00000100:00000001:2.0:1775776979.296769:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776979.296773:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776979.296811:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296817:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776979.296819:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296822:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.296824:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776979.296827:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296832:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776979.296837:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296841:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776979.296844:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296845:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.296847:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296849:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776979.296852:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776979.296855:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776979.296859:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776979.296861:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776979.296864:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776979.296870:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818240:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776979.296875:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776979.296877:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776976/real 1775776979] req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776979.296882:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776979.296883:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776979.296885:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069f8. 00000400:00000200:2.0:1775776979.296887:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8880 00000400:00000010:2.0:1775776979.296888:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8880. 00000100:00000001:2.0:1775776979.296890:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776979.296893:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776979.296898:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296901:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776979.296902:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296903:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.296904:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776979.296906:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296914:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776979.296916:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296920:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776979.296921:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296922:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.296923:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296923:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776979.296924:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776979.296925:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776979.296927:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776979.296929:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776979.296930:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776979.296934:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818368:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776979.296937:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776979.296939:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776976/real 1775776979] req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776979.296943:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776979.296944:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776979.296945:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306c60. 00000400:00000200:2.0:1775776979.296946:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8908 00000400:00000010:2.0:1775776979.296947:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8908. 00000100:00000001:2.0:1775776979.296947:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776979.296949:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776979.296957:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296961:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776979.296962:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296963:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.296964:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776979.296965:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296969:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776979.296970:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.296974:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776979.296975:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296976:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.296976:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.296977:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776979.296978:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776979.296979:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776979.296982:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776979.296983:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776979.296985:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776979.296988:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818496:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776979.296991:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776979.296992:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776976/real 1775776979] req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776979.297001:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776979.297002:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776979.297003:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306e70. 00000400:00000200:2.0:1775776979.297004:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b84c8 00000400:00000010:2.0:1775776979.297005:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b84c8. 00000100:00000001:2.0:1775776979.297006:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776979.297008:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776979.297011:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297015:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776979.297015:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297016:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.297017:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776979.297019:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297022:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776979.297023:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297027:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776979.297028:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297029:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.297029:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297030:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776979.297031:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776979.297033:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776979.297035:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776979.297036:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776979.297039:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776979.297044:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818624:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776979.297046:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776979.297049:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776976/real 1775776979] req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776979.297053:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776979.297053:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776979.297055:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306688. 00000400:00000200:2.0:1775776979.297056:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8aa0 00000400:00000010:2.0:1775776979.297057:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8aa0. 00000100:00000001:2.0:1775776979.297058:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776979.297059:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776979.297063:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297066:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776979.297067:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297068:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.297068:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776979.297070:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297073:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776979.297075:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297078:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776979.297079:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297080:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776979.297081:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297082:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776979.297083:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776979.297085:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776979.297087:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776979.297089:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776979.297091:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776979.297094:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099818112:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776979.297096:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297098:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776979.297099:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d880 x1862033099818240/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297104:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776979.297106:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776979.297110:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:2.0:1775776979.297115:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776979.297117:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776979.297119:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776979.297120:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776979.297123:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776979.297126:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776979.297127:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776979.297129:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d880. 00000100:00000001:2.0:1775776979.297132:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776979.297133:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297133:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776979.297135:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7dc00 x1862033099818368/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297139:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776979.297139:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776979.297141:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:2.0:1775776979.297142:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776979.297143:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776979.297144:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776979.297145:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776979.297146:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776979.297148:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776979.297148:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776979.297149:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7dc00. 00000100:00000001:2.0:1775776979.297150:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776979.297150:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297151:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776979.297153:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7df80 x1862033099818496/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297156:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776979.297157:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776979.297157:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:2.0:1775776979.297158:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776979.297159:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776979.297160:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776979.297161:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776979.297161:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776979.297162:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776979.297163:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776979.297163:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7df80. 00000100:00000001:2.0:1775776979.297164:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776979.297165:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297166:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776979.297167:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099818624/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297170:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776979.297171:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776979.297172:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:2.0:1775776979.297174:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776979.297174:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776979.297175:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776979.297176:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776979.297177:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776979.297178:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776979.297179:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776979.297180:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:2.0:1775776979.297181:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776979.297182:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297183:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776979.297184:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f480 x1862033099818112/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776979.297187:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776979.297188:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776979.297189:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:2.0:1775776979.297191:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776979.297192:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776979.297193:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776979.297194:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776979.297195:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776979.297197:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776979.297198:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776979.297199:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f480. 00000100:00000001:2.0:1775776979.297201:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776979.297202:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297204:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776979.297212:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776979.297215:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776979.297217:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776979.297218:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776979.996340:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776979.996350:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776979.996352:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776979.996361:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776979.996365:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776979.996371:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776979.996374:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776979.996377:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776979.996380:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776980.060058:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776980.060064:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776980.060069:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776980.060069:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776980.060073:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776980.060074:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776980.060076:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776980.060077:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776980.060149:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776980.060153:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776980.060156:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776980.188660:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776980.188669:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776980.188672:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776980.188679:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776980.188683:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776980.188688:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776980.188692:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776980.188695:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776980.188698:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776981.020095:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776981.020103:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776981.020106:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776981.020112:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776981.020117:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776981.020122:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776981.020124:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776981.020128:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776981.020131:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087089:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776981.087097:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087103:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776981.087107:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776981.087110:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776981.087115:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087118:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776981.087121:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087123:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087125:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087129:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776981.087130:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776981.087131:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776981.087133:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087134:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776981.087136:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087137:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087139:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087143:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776981.087144:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776981.087146:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776981.087147:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087149:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776981.087150:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087151:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776981.087162:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776981.087475:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776981.087479:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776981.087483:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776981.211925:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776981.211933:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776981.211934:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776981.211940:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776981.211943:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776981.211948:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776981.211950:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776981.211952:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776981.211955:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776981.339995:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776981.340008:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776981.340012:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776981.340017:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776981.340023:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776981.340041:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18696 00000400:00000200:0.0:1775776981.340054:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776981.340062:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776981.340066:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776981.340069:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.340072:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.340075:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.340078:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776981.340083:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776981.340085:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775776981.340088:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776981.340091:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775776981.340093:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775776981.340095:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775776981.340098:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.340100:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.340102:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.340104:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776981.340107:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775776981.340110:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776981.340113:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776981.340116:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776981.340120:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776981.340123:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.340125:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776981.340128:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776981.340135:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454df480. 00000020:00000040:0.0:1775776981.340140:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775776981.340145:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776981.340147:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.340150:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776981.340154:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776981.340158:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776981.340167:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775776981.340174:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776981.340187:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776981.340198:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776981.340204:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776981.340209:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033edb665] to pc [ptlrpcd_rcv+-1] req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.340262:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776981.340267:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776981.340271:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776981.340277:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.340285:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776981.340288:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.340292:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776981.340295:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776981.340298:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776981.340301:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099818752:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776981.340308:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776981.340314:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776981.340316:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.340319:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776981.340324:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776981.340327:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776981.340334:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776981.340338:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776981.340346:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776981.340356:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099818752, portal 25 00000100:00000001:2.0:1775776981.340359:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776981.340360:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741760640 : -114841967790976 : ffff978d454df480) 00000100:00000040:2.0:1775776981.340366:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.340374:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776981.340376:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776981.340379:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776981.340384:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099818752, offset 0 00000400:00000200:2.0:1775776981.340392:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776981.340413:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.340428:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776981.340433:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776981.340437:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776981.340441:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776981.340443:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776981.340447:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4627:4627:256:4294967295] 192.168.201.139@tcp LPNI seq info [4627:4627:8:4294967295] 00000400:00000200:2.0:1775776981.340450:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.340460:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776981.340466:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.340610:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776981.340630:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776981.340635:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776981.340641:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776981.340645:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776981.340649:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.340652:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776981.340660:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776981.340666:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776981.340674:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776981.340677:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776981.340682:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776981.340691:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776981.341567:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776981.341578:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776981.341593:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776981.341596:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776981.341599:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776981.341604:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776981.341611:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18696 00000400:00000200:0.0:1775776981.341620:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776981.341626:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776981.341630:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776981.341633:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.341635:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.341638:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.341640:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776981.341644:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776981.341645:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775776981.341648:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776981.341650:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775776981.341654:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775776981.341656:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775776981.341659:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.341661:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.341663:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.341665:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776981.341668:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775776981.341671:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776981.341673:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776981.341676:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776981.341685:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776981.341689:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.341692:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776981.341697:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776981.341700:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.341702:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776981.341710:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd880. 00000020:00000040:0.0:1775776981.341712:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775776981.341716:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776981.341718:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.341721:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776981.341724:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776981.341728:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776981.341735:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775776981.341741:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776981.341760:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776981.341772:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776981.341777:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776981.341815:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009caffe71] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776981.344042:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776981.344055:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776981.344060:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776981.344062:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776981.344066:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776981.344070:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776981.344078:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18696 00000400:00000200:0.0:1775776981.344088:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776981.344095:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776981.344098:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776981.344101:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344104:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.344107:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344109:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776981.344113:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776981.344114:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775776981.344118:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776981.344120:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775776981.344123:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775776981.344125:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775776981.344128:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344130:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.344132:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344134:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776981.344137:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775776981.344140:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344142:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776981.344146:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776981.344156:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776981.344159:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344161:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776981.344166:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776981.344169:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344171:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776981.344182:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc700. 00000020:00000040:0.0:1775776981.344186:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775776981.344190:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776981.344192:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344195:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776981.344197:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776981.344201:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776981.344210:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775776981.344216:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776981.344224:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776981.344234:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776981.344238:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776981.344243:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040c82de9] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776981.344253:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776981.344255:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776981.344260:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775776981.344261:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776981.344262:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776981.344265:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776981.344268:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18696 00000400:00000200:0.0:1775776981.344272:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776981.344275:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776981.344277:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776981.344281:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344282:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.344284:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344286:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776981.344288:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776981.344289:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775776981.344291:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776981.344293:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775776981.344295:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775776981.344297:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775776981.344300:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344301:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.344303:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344305:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776981.344307:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775776981.344309:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344311:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776981.344312:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776981.344315:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776981.344317:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344319:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776981.344321:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776981.344323:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344325:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776981.344328:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dce00. 00000020:00000040:0.0:1775776981.344330:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775776981.344332:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776981.344333:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344335:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776981.344336:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776981.344337:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776981.344341:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775776981.344344:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776981.344348:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776981.344355:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776981.344356:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776981.344359:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c8ee1924] to pc [ptlrpcd_rcv+-1] req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776981.344365:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776981.344367:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776981.344370:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775776981.344372:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776981.344373:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776981.344376:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776981.344378:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18696 00000400:00000200:0.0:1775776981.344381:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776981.344384:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776981.344386:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776981.344389:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344391:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.344392:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344394:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775776981.344396:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776981.344397:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775776981.344398:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776981.344400:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775776981.344402:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775776981.344404:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775776981.344407:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344408:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776981.344410:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344411:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775776981.344414:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775776981.344428:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775776981.344431:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776981.344432:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776981.344435:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776981.344436:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344438:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776981.344440:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775776981.344444:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344446:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776981.344451:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc380. 00000020:00000040:0.0:1775776981.344454:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775776981.344456:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776981.344458:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776981.344460:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776981.344462:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776981.344464:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776981.344470:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759d800. 00000100:00000001:2.0:1775776981.344471:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775776981.344475:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776981.344483:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775776981.344492:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776981.344496:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776981.344500:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776981.344506:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ea16ecb7] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776981.344518:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776981.344532:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18706) 00000100:00000001:2.0:1775776981.344532:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.344536:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776981.344556:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776981.344564:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776981.344568:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313370361 00000100:00000010:2.0:1775776981.344578:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bc60. 00000100:00000001:2.0:1775776981.344586:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776981.344594:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313370361 00000100:00000010:2.0:1775776981.344597:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bba0. 00000100:00000001:2.0:1775776981.344601:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776981.344606:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776981.344610:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.344613:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776981.344620:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776981/real 1775776981] req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776981.344634:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776981.344635:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776981.344640:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776981.344644:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776981.344647:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776981.344651:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776981.344654:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776981.344663:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.344670:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776981.344671:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.344673:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.344675:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776981.344679:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.344685:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776981.344689:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.344697:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776981.344700:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.344701:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.344702:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.344705:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776981.344707:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776981.344710:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776981.344714:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776981.344716:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776981.344720:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776981.344728:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099818752:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776981.344735:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.344737:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.344740:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454df480 x1862033099818752/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.344747:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776981.344749:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776981.344753:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776981.344756:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776981.344759:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776981.344760:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776981.344762:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776981.344764:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f400. 02000000:00000001:2.0:1775776981.344769:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776981.344770:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776981.344772:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454df480. 00000100:00000001:2.0:1775776981.344775:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776981.344776:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.344809:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.344816:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776981.344818:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.344821:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776981.344824:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776981.344825:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776981.344830:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.344838:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776981.344840:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.344843:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776981.344846:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776981.344848:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776981.344851:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818880:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776981.344855:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776981.344860:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776981.344862:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.344865:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776981.344869:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776981.344871:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776981.344876:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776981.344878:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776981.344883:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776981.344888:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099818880, portal 10 00000100:00000001:2.0:1775776981.344891:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776981.344892:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741753472 : -114841967798144 : ffff978d454dd880) 00000100:00000040:2.0:1775776981.344899:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.344905:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776981.344908:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776981.344910:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776981.344913:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099818880, offset 0 00000400:00000200:2.0:1775776981.344918:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776981.344924:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.344930:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776981.344934:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776981.344938:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776981.344941:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776981.344944:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776981.344948:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4628:4628:256:4294967295] 192.168.201.139@tcp LPNI seq info [4628:4628:8:4294967295] 00000400:00000200:2.0:1775776981.344953:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.344962:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776981.344967:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.344972:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776981.344978:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776981.344982:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776981.344987:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776981.344990:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776981.344995:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.344997:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776981.345001:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776981.345006:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776981.345009:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776981.345011:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776981.345013:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776981.345016:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345023:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.345026:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345032:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.345033:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776981.345036:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776981.345039:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776981.345042:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776981.345043:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313369097 00000100:00000001:2.0:1775776981.345046:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313370361 00000100:00000001:2.0:1775776981.345048:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313369097 00000100:00000001:2.0:1775776981.345050:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313370361 00000100:00000001:2.0:1775776981.345052:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776981.345054:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.345055:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776981.345058:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776981/real 1775776981] req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776981.345065:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776981.345067:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776981.345068:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776981.345070:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776981.345071:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776981.345073:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776981.345076:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776981.345083:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345089:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776981.345090:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.345092:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.345093:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776981.345096:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345101:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776981.345104:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345110:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776981.345112:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.345113:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.345114:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.345116:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776981.345118:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776981.345120:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776981.345123:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776981.345125:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776981.345129:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776981.345136:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099818880:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776981.345139:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776981.345143:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345154:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776981.345156:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.345158:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776981.345159:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776981.345161:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776981.345163:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819008:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776981.345166:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776981.345167:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776981.345169:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.345170:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776981.345173:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776981.345174:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776981.345177:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776981.345179:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776981.345181:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776981.345183:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099819008, portal 10 00000100:00000001:2.0:1775776981.345185:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776981.345186:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748992 : -114841967802624 : ffff978d454dc700) 00000100:00000040:2.0:1775776981.345190:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345197:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776981.345199:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776981.345200:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776981.345202:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099819008, offset 0 00000400:00000200:2.0:1775776981.345205:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776981.345208:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.345211:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776981.345214:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776981.345216:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776981.345218:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776981.345220:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776981.345223:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4629:4629:256:4294967295] 192.168.201.139@tcp LPNI seq info [4629:4629:8:4294967295] 00000400:00000200:2.0:1775776981.345227:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.345233:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776981.345237:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.345240:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776981.345243:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776981.345245:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776981.345249:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776981.345251:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776981.345254:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.345256:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776981.345258:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776981.345261:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776981.345264:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776981.345265:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776981.345267:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776981.345270:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345277:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.345280:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345286:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.345287:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776981.345289:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776981.345291:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776981.345294:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776981.345295:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313369098 00000100:00000001:2.0:1775776981.345297:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313370361 00000100:00000001:2.0:1775776981.345299:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313369098 00000100:00000001:2.0:1775776981.345301:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313370361 00000100:00000001:2.0:1775776981.345303:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776981.345305:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.345307:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776981.345309:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776981/real 1775776981] req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776981.345317:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776981.345318:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776981.345319:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776981.345321:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776981.345322:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776981.345324:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776981.345327:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776981.345335:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345341:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776981.345342:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.345343:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.345345:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776981.345348:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345353:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776981.345356:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345362:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776981.345364:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.345365:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.345366:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.345367:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776981.345369:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776981.345371:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776981.345374:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776981.345376:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776981.345379:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776981.345387:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819008:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776981.345390:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776981.345393:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.345399:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776981.345400:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.345402:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776981.345404:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776981.345405:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776981.345408:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819136:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776981.345411:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776981.345413:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776981.345414:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.346565:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776981.346575:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776981.346579:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776981.346586:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776981.346590:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776981.346595:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776981.346600:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099819136, portal 4 00000100:00000001:2.0:1775776981.346602:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776981.346604:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741750784 : -114841967800832 : ffff978d454dce00) 00000100:00000040:2.0:1775776981.346613:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.346620:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776981.346623:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776981.346625:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776981.346628:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099819136, offset 0 00000400:00000200:2.0:1775776981.346633:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776981.346639:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.346644:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776981.346648:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776981.346651:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776981.346654:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776981.346656:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776981.346659:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4630:4630:256:4294967295] 192.168.201.139@tcp LPNI seq info [4630:4630:8:4294967295] 00000400:00000200:2.0:1775776981.346663:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.346672:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776981.346677:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.346680:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776981.346685:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776981.346688:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776981.346692:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776981.346694:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776981.346698:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.346699:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776981.346704:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776981.346707:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776981.346710:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776981.346713:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776981.346715:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776981.346718:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.346726:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.346729:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.346735:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.346737:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776981.346742:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776981.346746:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776981.346750:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776981.346752:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313369108 00000100:00000001:2.0:1775776981.346755:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313370361 00000100:00000001:2.0:1775776981.346757:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313369108 00000100:00000001:2.0:1775776981.346759:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313370361 00000100:00000001:2.0:1775776981.346761:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776981.346763:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.346766:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776981.346769:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776981/real 1775776981] req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776981.346792:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776981.346794:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776981.346796:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776981.346799:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776981.346800:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776981.346803:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776981.346806:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776981.346814:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.346820:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776981.346822:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.346824:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.346826:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776981.346830:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.346837:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776981.346840:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.346848:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776981.346851:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.346852:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.346854:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.346856:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776981.346858:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776981.346862:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776981.346866:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776981.346868:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776981.346872:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776981.346879:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819136:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776981.346883:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776981.346887:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.346893:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776981.346895:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.346897:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776981.346899:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776981.346901:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776981.346904:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819264:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776981.346907:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776981.346909:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776981.346911:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776981.346913:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776981.346916:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776981.346917:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776981.346920:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776981.346924:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776981.346926:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776981.346929:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099819264, portal 4 00000100:00000001:2.0:1775776981.346931:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776981.346932:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741748096 : -114841967803520 : ffff978d454dc380) 00000100:00000040:2.0:1775776981.346937:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.346942:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776981.346944:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776981.346946:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776981.346948:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099819264, offset 0 00000400:00000200:2.0:1775776981.346952:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776981.346956:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.346959:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776981.346961:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776981.346963:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776981.346965:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776981.346967:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776981.346969:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4631:4631:256:4294967295] 192.168.201.139@tcp LPNI seq info [4631:4631:8:4294967295] 00000400:00000200:2.0:1775776981.346973:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776981.346978:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776981.346982:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.346985:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776981.346989:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776981.346991:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1775776981.346995:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776981.346997:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1775776981.347001:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776981.347002:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776981.347005:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776981.347007:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776981.347010:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776981.347011:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776981.347014:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776981.347017:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347025:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.347028:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777006 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347035:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.347036:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776981.347039:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776981.347042:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776981.347045:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776981.347046:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313369100 00000100:00000001:2.0:1775776981.347048:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313370361 00000100:00000001:2.0:1775776981.347050:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313369100 00000100:00000001:2.0:1775776981.347052:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313370361 00000100:00000001:2.0:1775776981.347053:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776981.347055:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.347056:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776981.347060:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776981/real 1775776981] req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776981.347066:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776981.347067:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776981.347069:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776981.347071:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776981.347072:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776981.347074:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776981.347077:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776981.347085:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347096:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776981.347097:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347099:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.347100:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776981.347103:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347109:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776981.347112:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347118:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776981.347121:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347122:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776981.347123:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347125:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776981.347127:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776981.347129:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776981.347132:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776981.347135:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776981.347138:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776981.347144:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819264:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776981.347148:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347150:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.347154:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd880 x1862033099818880/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347161:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776981.347164:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776981.347167:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776981.347170:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776981.347172:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776981.347174:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776981.347176:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776981.347178:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:2.0:1775776981.347182:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776981.347184:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776981.347186:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd880. 00000100:00000001:2.0:1775776981.347189:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776981.347190:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347192:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.347195:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc700 x1862033099819008/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347201:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776981.347203:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776981.347204:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776981.347206:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776981.347207:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776981.347208:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776981.347210:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776981.347211:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:2.0:1775776981.347213:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776981.347214:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776981.347216:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc700. 00000100:00000001:2.0:1775776981.347218:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776981.347219:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347220:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.347223:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dce00 x1862033099819136/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347229:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776981.347231:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776981.347232:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504400. 02000000:00000001:2.0:1775776981.347234:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776981.347235:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776981.347236:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776981.347238:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776981.347239:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775776981.347242:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776981.347243:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776981.347244:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dce00. 00000100:00000001:2.0:1775776981.347245:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776981.347247:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347248:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776981.347251:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc380 x1862033099819264/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777006 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776981.347257:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776981.347258:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776981.347259:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507400. 02000000:00000001:2.0:1775776981.347261:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776981.347262:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776981.347263:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776981.347265:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776981.347266:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759d800. 02000000:00000001:2.0:1775776981.347268:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776981.347269:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776981.347270:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc380. 00000100:00000001:2.0:1775776981.347273:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776981.347273:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347276:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776981.347283:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776981.347285:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776981.347287:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776981.347288:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776982.044163:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776982.044183:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776982.044185:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776982.044197:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776982.044201:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776982.044212:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776982.044215:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776982.044218:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776982.044221:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776982.108062:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776982.108068:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776982.108074:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776982.108075:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776982.108078:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776982.108079:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776982.108092:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776982.108093:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776982.108205:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776982.108221:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776982.108224:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776982.235896:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776982.235902:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776982.235904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776982.235910:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776982.235913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776982.235917:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776982.235919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776982.235922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776982.235924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776982.620028:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313370361, at 4313370368 00000100:00000010:0.0:1775776982.620101:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bc60. 00000100:00000040:1.0:1775776982.620182:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313370361, at 4313370368 00000100:00000010:1.0:1775776982.620188:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bba0. 00000100:00000001:3.0:1775776983.070533:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776983.070540:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776983.070542:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776983.070547:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776983.070551:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776983.070555:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776983.070557:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776983.070559:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776983.070562:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776983.131887:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776983.131891:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776983.131896:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776983.131897:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776983.131900:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776983.131901:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776983.131904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776983.131905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776983.134650:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776983.134658:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776983.134663:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776983.260644:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776983.260651:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776983.260653:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776983.260658:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776983.260661:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776983.260664:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776983.260666:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776983.260669:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776983.260672:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776984.091908:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776984.091914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776984.091952:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776984.091957:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776984.091960:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776984.091964:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776984.091967:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776984.091969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776984.091972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776984.157534:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776984.157539:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776984.157544:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776984.157546:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776984.157549:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776984.157550:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776984.157553:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776984.157554:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776984.157656:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776984.157659:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776984.157662:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776984.283899:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776984.283906:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776984.283908:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776984.283914:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776984.283917:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776984.283920:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776984.283923:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776984.283925:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776984.283928:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776985.116028:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776985.116036:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776985.116038:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776985.116043:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776985.116047:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776985.116050:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776985.116053:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776985.116055:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776985.116058:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776985.180200:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776985.180206:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776985.180213:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776985.180214:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776985.180218:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776985.180219:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776985.180222:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776985.180223:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776985.181031:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776985.181034:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776985.181038:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776985.307914:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776985.307920:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776985.307922:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776985.307926:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776985.307929:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776985.307933:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776985.307936:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776985.307937:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776985.307940:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776986.140524:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776986.140538:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776986.140540:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776986.140545:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776986.140549:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776986.140553:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776986.140556:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776986.140558:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776986.140561:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776986.205598:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776986.205619:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776986.205624:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776986.205625:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776986.205629:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776986.205629:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776986.205632:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776986.205633:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776986.205713:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776986.207379:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776986.207386:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776986.331880:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776986.331887:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776986.331888:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776986.331893:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776986.331897:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776986.331900:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776986.331910:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776986.331912:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776986.331915:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776986.460110:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776986.460120:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776986.460123:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776986.460126:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776986.460131:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776986.460137:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18701 00000400:00000200:1.0:1775776986.460166:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776986.460172:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776986.460176:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776986.460179:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460183:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.460185:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460197:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776986.460201:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776986.460204:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775776986.460206:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776986.460220:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775776986.460223:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775776986.460226:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775776986.460229:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460232:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.460233:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460235:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776986.460238:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775776986.460242:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460245:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776986.460248:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776986.460251:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776986.460253:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.460255:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776986.460258:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776986.460267:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775776986.460270:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775776986.460274:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776986.460276:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.460279:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776986.460282:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776986.460286:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776986.460295:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776986.460302:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776986.460311:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776986.460322:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776986.460342:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776986.460348:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776986.460376:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776986.460379:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776986.460382:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776986.460384:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776986.460385:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776986.460388:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776986.460390:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18701 00000400:00000200:1.0:1775776986.460394:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776986.460397:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776986.460399:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776986.460402:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460403:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.460405:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460407:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776986.460408:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776986.460410:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775776986.460411:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776986.460413:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775776986.460414:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775776986.460416:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775776986.460419:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460420:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.460422:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460423:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776986.460425:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775776986.460437:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776986.460439:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776986.460441:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776986.460446:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776986.460448:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.460449:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776986.460454:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776986.460456:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.460457:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776986.460461:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775776986.460463:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775776986.460465:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776986.460466:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.460468:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776986.460469:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776986.460470:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776986.460474:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775776986.460477:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776986.460481:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776986.460488:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776986.460490:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776986.460493:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.460708:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776986.460713:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776986.460716:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776986.460722:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.460729:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776986.460732:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.460735:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776986.460738:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776986.460740:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776986.460744:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099819392:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776986.460748:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776986.460751:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776986.460753:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.460756:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776986.460761:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776986.460763:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776986.460768:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776986.460822:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776986.460831:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776986.460838:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099819392, portal 25 00000100:00000001:2.0:1775776986.460841:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776986.460842:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609984 : -114841421941632 : ffff978d65d6f480) 00000100:00000040:2.0:1775776986.460848:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.460855:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776986.460858:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776986.460861:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776986.460864:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099819392, offset 0 00000400:00000200:2.0:1775776986.460886:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776986.460893:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.460900:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776986.460905:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776986.460909:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776986.460913:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776986.460916:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776986.460920:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4632:4632:256:4294967295] 192.168.201.139@tcp LPNI seq info [4632:4632:8:4294967295] 00000400:00000200:2.0:1775776986.460924:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.460935:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776986.460941:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.460947:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776986.460953:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776986.460957:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776986.460962:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776986.460966:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776986.460970:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.460973:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776986.460979:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776986.460984:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776986.460989:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776986.460991:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776986.460995:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776986.460999:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.461020:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.461024:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.461031:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.461034:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776986.461039:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776986.461043:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776986.461046:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313375478 00000100:00000010:2.0:1775776986.461051:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b1e0. 00000100:00000001:2.0:1775776986.461057:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776986.461065:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313375478 00000100:00000010:2.0:1775776986.461068:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b5a0. 00000100:00000001:2.0:1775776986.461071:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776986.461074:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776986.461076:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.461078:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776986.461082:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776986/real 1775776986] req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776986.461098:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776986.461099:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776986.461102:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776986.461104:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776986.461106:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776986.461108:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776986.461111:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776986.461120:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.461126:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776986.461128:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.461131:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.461133:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776986.461136:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.461143:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776986.461147:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.461154:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776986.461157:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.461158:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.461159:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.461162:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776986.461163:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776986.461167:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776986.461171:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776986.461174:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776986.461178:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776986.461187:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099819392:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776986.461192:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.461194:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.461197:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f480 x1862033099819392/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.461204:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776986.461207:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776986.461210:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776986.461213:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776986.461214:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776986.461215:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776986.461218:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776986.461220:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3ac00. 02000000:00000001:2.0:1775776986.461223:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776986.461225:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776986.461227:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f480. 00000100:00000001:2.0:1775776986.461230:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776986.461231:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.461233:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.461239:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776986.461241:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.461244:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776986.461245:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.461687:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776986.461691:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776986.461694:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776986.461696:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776986.461697:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776986.461700:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776986.461703:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18701 00000400:00000200:1.0:1775776986.461707:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776986.461710:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776986.461712:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776986.461716:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461718:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.461720:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461722:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776986.461724:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776986.461725:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775776986.461727:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776986.461729:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775776986.461730:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775776986.461732:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775776986.461735:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461737:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.461739:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461741:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776986.461743:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775776986.461746:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461748:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776986.461750:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776986.461753:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776986.461755:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.461757:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776986.461759:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776986.461762:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.461763:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776986.461767:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775776986.461769:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775776986.461772:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776986.461773:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.461775:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776986.461776:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776986.461804:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776986.461810:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775776986.461815:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776986.461820:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776986.461827:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776986.461830:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776986.461833:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776986.461840:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776986.461843:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776986.461847:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775776986.461848:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776986.461849:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776986.461852:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776986.461854:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18701 00000400:00000200:1.0:1775776986.461858:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776986.461861:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776986.461863:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776986.461866:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461868:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.461869:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461871:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776986.461874:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776986.461875:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775776986.461877:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776986.461878:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775776986.461880:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775776986.461882:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775776986.461885:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461887:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.461889:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461890:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776986.461893:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775776986.461895:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461897:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776986.461899:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776986.461902:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776986.461903:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.461905:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776986.461907:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776986.461909:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.461911:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776986.461914:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775776986.461916:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775776986.461919:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776986.461920:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.461922:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776986.461923:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776986.461925:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776986.461928:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775776986.461931:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776986.461935:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776986.461942:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776986.461944:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776986.461947:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776986.461954:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776986.461956:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776986.461959:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775776986.461960:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776986.461962:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776986.461965:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776986.461967:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18701 00000400:00000200:1.0:1775776986.461970:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775776986.461973:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776986.461975:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776986.461977:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461980:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.461981:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461983:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775776986.461985:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776986.461986:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775776986.461988:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776986.461990:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775776986.461992:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775776986.461994:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775776986.461997:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.461999:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776986.462000:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775776986.462014:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775776986.462016:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775776986.462019:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775776986.462022:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776986.462023:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776986.462026:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776986.462028:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.462030:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776986.462032:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775776986.462034:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.462036:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776986.462040:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775776986.462042:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775776986.462044:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776986.462045:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776986.462047:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776986.462048:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776986.462050:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776986.462053:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775776986.462056:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776986.462060:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776986.462067:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776986.462069:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776986.462073:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776986.462080:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776986.462085:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18711) 00000100:00000001:2.0:1775776986.462435:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776986.462438:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776986.462439:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776986.462444:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.462452:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776986.462454:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.462457:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776986.462459:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776986.462461:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776986.462464:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819520:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776986.462468:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776986.462470:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776986.462472:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.462474:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776986.462478:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776986.462480:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776986.462484:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776986.462488:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776986.462492:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776986.462614:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099819520, portal 10 00000100:00000001:2.0:1775776986.462617:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776986.462618:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287596544 : -114841421955072 : ffff978d65d6c000) 00000100:00000040:2.0:1775776986.462625:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.462632:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776986.462634:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776986.462637:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776986.462640:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099819520, offset 0 00000400:00000200:2.0:1775776986.462645:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776986.462650:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.462654:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776986.462658:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776986.462660:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776986.462663:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776986.462665:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776986.462668:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4633:4633:256:4294967295] 192.168.201.139@tcp LPNI seq info [4633:4633:8:4294967295] 00000400:00000200:2.0:1775776986.462672:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.462680:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776986.462684:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.462688:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776986.462693:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776986.462697:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776986.462701:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776986.462704:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776986.462708:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.462710:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776986.462713:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776986.462717:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776986.462720:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776986.462722:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776986.462725:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776986.462728:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.462737:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.462740:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.462747:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.462748:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776986.462752:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776986.462756:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776986.462759:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776986.462761:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313374214 00000100:00000001:2.0:1775776986.462763:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313375478 00000100:00000001:2.0:1775776986.462765:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313374214 00000100:00000001:2.0:1775776986.462767:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313375478 00000100:00000001:2.0:1775776986.462769:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776986.462771:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.462774:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776986.462778:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776986/real 1775776986] req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776986.462897:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776986.462899:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776986.462901:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776986.462903:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776986.462905:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776986.462907:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776986.462911:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776986.462919:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.462925:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776986.462926:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.462928:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.462930:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776986.462933:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.462940:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776986.462943:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.462950:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776986.462952:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.462954:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.462955:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.462957:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776986.462959:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776986.462962:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776986.462965:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776986.462967:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776986.462971:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776986.462984:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819520:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776986.462988:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776986.462991:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.462998:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776986.463000:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.463001:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776986.463075:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776986.463078:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776986.463081:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819648:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776986.463085:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776986.463087:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776986.463089:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.463090:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776986.463094:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776986.463096:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776986.463099:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776986.463102:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776986.463114:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776986.463117:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099819648, portal 10 00000100:00000001:2.0:1775776986.463119:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776986.463120:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287600128 : -114841421951488 : ffff978d65d6ce00) 00000100:00000040:2.0:1775776986.463126:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463132:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776986.463134:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776986.463136:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776986.463138:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099819648, offset 0 00000400:00000200:2.0:1775776986.463142:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776986.463146:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.463150:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776986.463152:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776986.463155:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776986.463157:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776986.463160:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776986.463163:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4634:4634:256:4294967295] 192.168.201.139@tcp LPNI seq info [4634:4634:8:4294967295] 00000400:00000200:2.0:1775776986.463166:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.463173:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776986.463177:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.463180:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776986.463184:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776986.463186:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776986.463190:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776986.463192:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776986.463196:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.463197:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776986.463200:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776986.463204:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776986.463206:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776986.463208:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776986.463210:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776986.463213:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463220:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.463223:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463230:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.463231:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776986.463234:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776986.463236:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775776986.463239:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776986.463241:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313374216 00000100:00000001:2.0:1775776986.463244:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313375478 00000100:00000001:2.0:1775776986.463246:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313374216 00000100:00000001:2.0:1775776986.463248:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313375478 00000100:00000001:2.0:1775776986.463250:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776986.463252:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.463254:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776986.463257:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776986/real 1775776986] req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776986.463264:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776986.463266:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776986.463267:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776986.463269:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776986.463271:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776986.463272:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776986.463275:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776986.463283:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463290:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776986.463291:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.463293:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.463294:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776986.463297:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463304:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776986.463307:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463314:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776986.463316:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.463318:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.463319:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.463321:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776986.463322:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776986.463325:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776986.463400:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776986.463402:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776986.463407:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776986.463414:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819648:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776986.463418:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776986.463421:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463555:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776986.463556:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.463559:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776986.463561:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776986.463563:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776986.463566:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819776:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776986.463569:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776986.463571:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776986.463573:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.463574:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776986.463578:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776986.463579:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776986.463583:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776986.463586:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776986.463589:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776986.463592:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099819776, portal 4 00000100:00000001:2.0:1775776986.463594:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776986.463595:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287608192 : -114841421943424 : ffff978d65d6ed80) 00000100:00000040:2.0:1775776986.463600:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463607:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776986.463609:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776986.463611:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776986.463613:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099819776, offset 0 00000400:00000200:2.0:1775776986.463617:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776986.463620:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.463624:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776986.463626:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776986.463629:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776986.463631:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776986.463633:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776986.463636:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4635:4635:256:4294967295] 192.168.201.139@tcp LPNI seq info [4635:4635:8:4294967295] 00000400:00000200:2.0:1775776986.463640:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.463646:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776986.463651:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.463653:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776986.463657:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776986.463659:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776986.463663:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776986.463666:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776986.463670:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.463671:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776986.463674:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776986.463677:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776986.463680:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776986.463681:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776986.463684:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776986.463687:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463700:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.463702:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463708:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.463709:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776986.463712:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776986.463715:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776986.463718:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776986.463720:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313374225 00000100:00000001:2.0:1775776986.463722:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313375478 00000100:00000001:2.0:1775776986.463724:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313374225 00000100:00000001:2.0:1775776986.463726:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313375478 00000100:00000001:2.0:1775776986.463728:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776986.463730:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.463731:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776986.463734:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776986/real 1775776986] req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776986.463741:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776986.463742:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776986.463744:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776986.463746:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776986.463747:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776986.463749:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776986.463752:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776986.463759:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463764:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776986.463765:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.463767:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.463768:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776986.463771:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463777:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776986.463921:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463929:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776986.463931:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.463933:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.463934:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.463935:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776986.463937:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776986.463939:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776986.463942:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776986.463944:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776986.463947:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776986.463954:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819776:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776986.463957:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776986.463960:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.463965:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776986.463967:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.463969:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776986.463970:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776986.463972:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776986.463975:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819904:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776986.463977:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776986.463979:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776986.463980:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776986.463982:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776986.463985:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776986.463987:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776986.463990:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776986.463993:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776986.463996:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776986.463999:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099819904, portal 4 00000100:00000001:2.0:1775776986.464001:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776986.464002:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601024 : -114841421950592 : ffff978d65d6d180) 00000100:00000040:2.0:1775776986.464088:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464095:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776986.464097:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776986.464100:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776986.464103:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099819904, offset 0 00000400:00000200:2.0:1775776986.464107:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776986.464110:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.464114:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776986.464117:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776986.464119:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776986.464122:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776986.464124:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776986.464127:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4636:4636:256:4294967295] 192.168.201.139@tcp LPNI seq info [4636:4636:8:4294967295] 00000400:00000200:2.0:1775776986.464131:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776986.464136:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776986.464140:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.464144:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776986.464147:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776986.464150:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1775776986.464153:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776986.464155:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1775776986.464158:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776986.464160:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776986.464163:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776986.464166:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776986.464168:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776986.464169:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776986.464172:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776986.464174:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464181:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.464184:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777011 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464189:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.464190:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776986.464193:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776986.464195:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776986.464198:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776986.464200:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313374217 00000100:00000001:2.0:1775776986.464203:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313375478 00000100:00000001:2.0:1775776986.464205:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313374217 00000100:00000001:2.0:1775776986.464207:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313375478 00000100:00000001:2.0:1775776986.464209:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776986.464210:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.464212:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776986.464215:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776986/real 1775776986] req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776986.464222:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776986.464224:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776986.464226:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776986.464228:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776986.464229:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776986.464231:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776986.464234:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776986.464241:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464247:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776986.464248:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464250:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.464251:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776986.464254:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464260:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776986.464264:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464271:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776986.464272:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464274:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776986.464275:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464276:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776986.464278:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776986.464281:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776986.464284:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776986.464286:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776986.464289:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776986.464302:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099819904:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776986.464306:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464308:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.464311:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c000 x1862033099819520/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464317:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776986.464320:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776986.464322:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505800. 02000000:00000001:2.0:1775776986.464325:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776986.464400:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776986.464401:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776986.464404:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776986.464406:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38000. 02000000:00000001:2.0:1775776986.464410:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776986.464411:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776986.464413:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c000. 00000100:00000001:2.0:1775776986.464415:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776986.464417:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464418:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.464421:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ce00 x1862033099819648/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464565:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776986.464568:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776986.464570:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776986.464573:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776986.464574:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776986.464575:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776986.464577:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776986.464578:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3ac00. 02000000:00000001:2.0:1775776986.464580:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776986.464581:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776986.464583:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ce00. 00000100:00000001:2.0:1775776986.464586:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776986.464587:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464589:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.464592:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ed80 x1862033099819776/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464598:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776986.464600:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776986.464601:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776986.464603:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776986.464604:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776986.464605:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776986.464607:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776986.464608:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39c00. 02000000:00000001:2.0:1775776986.464610:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776986.464611:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776986.464612:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ed80. 00000100:00000001:2.0:1775776986.464615:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776986.464616:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464617:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776986.464620:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d180 x1862033099819904/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777011 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776986.464625:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776986.464626:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776986.464627:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505400. 02000000:00000001:2.0:1775776986.464629:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776986.464630:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776986.464630:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776986.464632:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776986.464633:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a000. 02000000:00000001:2.0:1775776986.464635:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776986.464636:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776986.464637:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d180. 00000100:00000001:2.0:1775776986.464639:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776986.464640:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464642:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776986.464648:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776986.464650:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776986.464653:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776986.464654:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776987.165441:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776987.165447:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776987.165449:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776987.165454:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776987.165457:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776987.165460:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776987.165462:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776987.165464:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776987.165467:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776987.227895:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776987.227900:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776987.227904:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776987.227905:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776987.227907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776987.227908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776987.227910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776987.227911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776987.227965:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776987.227969:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776987.227972:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776987.355890:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776987.355897:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776987.355899:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776987.355905:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776987.355909:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776987.355913:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776987.355916:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776987.355918:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776987.355920:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776987.739938:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313375478, at 4313375488 00000100:00000010:0.0:1775776987.739954:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b1e0. 00000100:00000040:1.0:1775776987.740188:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313375478, at 4313375488 00000100:00000010:1.0:1775776987.740193:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b5a0. 00000100:00000001:3.0:1775776988.187936:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776988.187942:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776988.187945:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776988.187951:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776988.187954:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776988.187958:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776988.187960:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776988.187974:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776988.187977:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776988.251969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776988.251975:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776988.251980:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776988.251981:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776988.251985:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776988.251986:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776988.251989:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776988.251990:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776988.270542:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776988.270559:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776988.270564:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776988.380200:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776988.380208:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776988.380210:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776988.380216:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776988.380219:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776988.380223:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776988.380226:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776988.380228:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776988.380231:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776989.211944:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776989.211951:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776989.211953:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776989.211958:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776989.211961:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776989.211965:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776989.211967:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776989.211969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776989.211972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776989.276921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776989.276929:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776989.276931:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776989.276934:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776989.276939:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776989.276941:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776989.276943:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776989.276946:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776989.276950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776989.276952:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776989.276955:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776989.276957:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776989.276960:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776989.276961:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776989.277051:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776989.277054:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776989.277057:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776989.428941:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776989.428948:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776989.428951:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776989.428956:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776989.428960:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776989.428964:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776989.428968:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776989.428970:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776989.428973:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776990.235968:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776990.235975:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776990.235977:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776990.235983:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776990.235987:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776990.235991:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776990.235994:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776990.235997:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776990.236000:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776990.299905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776990.299912:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776990.299918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776990.299919:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776990.299922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776990.299924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776990.299927:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776990.299928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776990.301487:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776990.301493:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776990.301497:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776990.554730:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776990.554738:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776990.554740:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776990.554747:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776990.554751:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776990.554756:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776990.554759:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776990.554762:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776990.554765:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.260011:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776991.260017:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776991.260019:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776991.260025:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776991.260028:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776991.260033:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776991.260035:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776991.260037:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776991.260040:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776991.323910:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776991.323920:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776991.323922:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776991.323925:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323928:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776991.323930:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323932:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323934:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323938:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776991.323939:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776991.323940:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776991.323942:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323943:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776991.323944:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323945:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323947:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323949:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776991.323951:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776991.323952:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776991.323953:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323954:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776991.323955:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323956:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776991.323958:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776991.324017:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776991.324021:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776991.324024:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776991.579919:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776991.579924:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776991.579927:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776991.579932:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776991.579936:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776991.579940:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776991.579942:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776991.579944:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776991.579947:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776991.580437:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776991.580443:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776991.580445:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776991.580449:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776991.580453:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776991.580459:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18706 00000400:00000200:3.0:1775776991.580469:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776991.580476:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776991.580945:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776991.580951:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.580955:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776991.580957:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.580959:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776991.580963:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776991.580965:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776991.580968:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776991.580971:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776991.580973:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776991.580975:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776991.580978:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.580980:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776991.580982:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.580984:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776991.580987:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776991.580990:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776991.580992:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776991.580995:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776991.580998:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776991.581001:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.581031:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776991.581033:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776991.581042:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775776991.581046:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776991.581049:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776991.581051:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.581054:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776991.581056:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776991.581061:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776991.581068:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775776991.581075:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776991.581084:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776991.581095:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776991.581100:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776991.581105:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776991.581133:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776991.581136:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776991.581140:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776991.581141:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776991.581143:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776991.581146:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776991.581148:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18706 00000400:00000200:3.0:1775776991.581153:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776991.581156:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776991.581158:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776991.581161:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.581163:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776991.581165:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.581167:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776991.581170:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776991.581171:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775776991.581173:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776991.581174:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775776991.581176:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775776991.581178:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776991.581181:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.581183:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776991.581185:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.581187:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776991.581189:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775776991.581191:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776991.581193:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776991.581195:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776991.581200:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776991.581202:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.581204:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776991.581208:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776991.581210:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.581213:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776991.581216:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775776991.581218:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775776991.581221:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776991.581222:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.581223:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776991.581225:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776991.581226:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776991.581229:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776991.581232:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776991.581246:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776991.581254:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776991.581257:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.581262:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776991.581262:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.581267:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776991.581270:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776991.581276:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776991.581279:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776991.581280:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776991.581285:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776991.581288:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776991.581291:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776991.581297:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776991.581297:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776991.581300:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776991.581303:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18706 02000000:00000001:2.0:1775776991.581305:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776991.581309:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1775776991.581309:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000001:2.0:1775776991.581314:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776991.581314:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776991.581318:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775776991.581320:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099820032:192.168.201.139@tcp:250:kworker.0 00000100:00000040:3.0:1775776991.581325:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776991.581327:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776991.581329:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775776991.581331:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775776991.581332:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000001:2.0:1775776991.581335:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.581336:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:2.0:1775776991.581339:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776991.581344:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776991.581346:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776991.581352:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776991.581356:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776991.581362:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776991.581368:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099820032, portal 25 00000100:00000001:2.0:1775776991.581370:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776991.581372:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751252608 : -114841958299008 : ffff978d45deca80) 00000100:00000040:2.0:1775776991.581456:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.581465:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776991.581468:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776991.581471:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776991.581475:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099820032, offset 0 00000400:00000200:2.0:1775776991.581482:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776991.581489:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776991.581495:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776991.581499:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776991.581503:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776991.581507:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776991.581510:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776991.581514:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4637:4637:256:4294967295] 192.168.201.139@tcp LPNI seq info [4637:4637:8:4294967295] 00000400:00000200:2.0:1775776991.581519:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776991.581529:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776991.581534:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776991.581540:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776991.581546:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776991.581550:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776991.581555:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776991.581559:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1775776991.581563:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776991.581565:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776991.581588:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776991.581593:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776991.581598:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776991.581600:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776991.581604:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776991.581608:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.581618:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.581621:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.581629:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.581630:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776991.581635:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776991.581639:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776991.581641:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313380598 00000100:00000010:2.0:1775776991.581646:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775776991.581652:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775776991.581658:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313380598 00000100:00000010:2.0:1775776991.581661:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b7e0. 00000100:00000001:2.0:1775776991.581664:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776991.581667:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776991.581669:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.581672:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776991.581675:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776991/real 1775776991] req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776991.581691:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776991.581693:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776991.581695:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776991.581697:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776991.581699:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776991.581701:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776991.581704:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776991.581712:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.581719:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776991.581720:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.581722:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.581724:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776991.581727:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.581734:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776991.581738:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.581745:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776991.581748:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.581749:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.581750:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.581752:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776991.581754:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776991.581757:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776991.581760:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776991.581762:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776991.581766:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776991.581774:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099820032:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776991.582276:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.582279:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.582284:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deca80 x1862033099820032/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582291:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776991.582294:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776991.582296:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776991.582299:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776991.582301:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776991.582303:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776991.582305:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776991.582307:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:2.0:1775776991.582311:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776991.582313:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776991.582315:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deca80. 00000100:00000001:2.0:1775776991.582318:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776991.582319:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.582321:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.582328:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776991.582329:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.582332:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776991.582333:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776991.582334:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776991.582338:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582345:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776991.582346:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776991.582348:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776991.582350:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776991.582352:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776991.582355:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820160:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776991.582358:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776991.582360:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776991.582362:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776991.582364:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776991.582368:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776991.582370:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776991.582375:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776991.582457:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776991.582462:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776991.582466:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099820160, portal 10 00000100:00000001:2.0:1775776991.582468:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776991.582469:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751259776 : -114841958291840 : ffff978d45dee680) 00000100:00000040:2.0:1775776991.582476:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582483:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776991.582486:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776991.582488:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776991.582490:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099820160, offset 0 00000400:00000200:2.0:1775776991.582494:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776991.582499:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776991.582503:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776991.582507:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776991.582510:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776991.582512:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776991.582515:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776991.582518:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4638:4638:256:4294967295] 192.168.201.139@tcp LPNI seq info [4638:4638:8:4294967295] 00000400:00000200:2.0:1775776991.582522:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776991.582530:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776991.582534:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776991.582537:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776991.582542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776991.582546:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776991.582550:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776991.582553:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1775776991.582556:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776991.582558:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776991.582561:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776991.582565:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776991.582568:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776991.582570:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776991.582572:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776991.582576:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582584:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.582587:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582594:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.582595:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776991.582598:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776991.582601:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775776991.582604:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776991.582606:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313379333 00000100:00000001:2.0:1775776991.582609:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313380598 00000100:00000001:2.0:1775776991.582611:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313379333 00000100:00000001:2.0:1775776991.582613:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313380598 00000100:00000001:2.0:1775776991.582615:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776991.582617:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.582619:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776991.582622:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776991/real 1775776991] req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776991.582630:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776991.582631:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776991.582633:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776991.582635:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776991.582636:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776991.582638:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776991.582641:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776991.582649:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582655:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776991.582656:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.582658:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.582659:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776991.582662:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582669:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776991.582674:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582681:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1775776991.582685:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775776991.582686:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775776991.582688:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000100:00000001:2.0:1775776991.582689:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.582692:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775776991.582693:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1775776991.582694:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776991.582697:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:3.0:1775776991.582697:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775776991.582702:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00080000:2.0:1775776991.582703:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1775776991.582707:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00080000:2.0:1775776991.582709:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776991.582712:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776991.582715:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776991.582723:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820160:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776991.582727:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.582729:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.582732:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee680 x1862033099820160/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.582738:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776991.582739:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776991.582741:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776991.582743:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776991.582744:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776991.582745:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775776991.582747:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776991.582748:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:2.0:1775776991.582751:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776991.582752:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776991.582753:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee680. 00000100:00000001:2.0:1775776991.582756:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776991.582757:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.582759:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.582762:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776991.582763:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.582765:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776991.582766:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776991.582864:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.582867:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776991.582869:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.582870:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776991.582873:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775776991.582876:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776991.582879:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776991.582881:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776991.582886:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776991.582888:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.582889:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776991.582893:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776991.582895:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.582897:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776991.582901:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775776991.582903:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775776991.582906:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776991.582908:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.582910:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776991.582911:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776991.582914:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776991.582918:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776991.582922:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776991.582927:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776991.582935:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776991.582938:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776991.582942:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776991.582960:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776991.582964:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776991.582967:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775776991.582968:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776991.582970:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776991.582973:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776991.582975:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18706 00000400:00000200:3.0:1775776991.582980:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776991.583248:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776991.583251:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776991.583252:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776991.583257:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583265:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776991.583266:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776991.583268:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776991.583270:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776991.583272:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776991.583275:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820288:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776991.583279:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000400:00000200:3.0:1775776991.583280:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 02000000:00000001:2.0:1775776991.583281:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776991.583284:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776991.583285:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1775776991.583288:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1775776991.583291:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 02000000:00000010:2.0:1775776991.583292:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 00000100:00000001:3.0:1775776991.583296:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1775776991.583297:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776991.583301:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776991.583304:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776991.583305:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000010:2.0:1775776991.583308:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000020:00000001:3.0:1775776991.583309:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776991.583312:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000400:00000010:2.0:1775776991.583314:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000020:00000001:3.0:1775776991.583314:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776991.583317:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000100:00000200:2.0:1775776991.583321:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099820288, portal 10 00000020:00000001:3.0:1775776991.583321:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776991.583325:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776991.583326:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776991.583328:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751253504 : -114841958298112 : ffff978d45dece00) 00000100:00000040:3.0:1775776991.583331:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.583334:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776991.583337:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776991.583339:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.583344:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776991.583349:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00000001:2.0:1775776991.583351:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776991.583354:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000040:2.0:1775776991.583356:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000100:00000001:3.0:1775776991.583360:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776991.583361:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 02000000:00000001:3.0:1775776991.583364:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:2.0:1775776991.583365:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099820288, offset 0 02000000:00000001:3.0:1775776991.583368:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1775776991.583372:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 02000000:00000001:3.0:1775776991.583374:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.583377:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776991.583379:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776991.583382:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.583383:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776991.583388:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775776991.583390:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776991.583393:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776991.583394:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776991.583396:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776991.583397:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776991.583399:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776991.583403:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775776991.583407:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776991.583412:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776991.583419:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776991.583422:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776991.583425:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776991.583433:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776991.583435:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776991.583438:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776991.583439:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776991.583441:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776991.583443:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776991.583445:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18706 00000400:00000200:3.0:1775776991.583450:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776991.583453:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776991.583455:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775776991.583459:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775776991.583461:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.583465:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776991.583467:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000040:3.0:1775776991.583470:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776991.583473:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775776991.583474:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776991.583478:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000020:00000001:3.0:1775776991.583478:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776991.583481:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000400:00000200:2.0:1775776991.583483:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1775776991.583484:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1775776991.583488:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000020:00000040:3.0:1775776991.583488:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776991.583493:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000400:00000200:2.0:1775776991.583494:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4639:4639:256:4294967295] 192.168.201.139@tcp LPNI seq info [4639:4639:8:4294967295] 00000020:00000001:3.0:1775776991.583499:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000400:00000200:2.0:1775776991.583503:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775776991.583507:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776991.583511:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776991.583516:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776991.583517:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000001:3.0:1775776991.583521:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776991.583525:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000800:00000200:2.0:1775776991.583528:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00080000:3.0:1775776991.583531:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000800:00000010:2.0:1775776991.583534:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000100:00000001:3.0:1775776991.583535:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776991.583538:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776991.583541:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1775776991.583542:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:3.0:1775776991.583546:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776991.583547:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1775776991.583550:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776991.583553:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000800:00000200:2.1:1775776991.583554:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000008:00000001:3.0:1775776991.583557:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776991.583559:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1775776991.583561:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775776991.583565:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00000010:3.0:1775776991.583567:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000800:00000010:2.0:1775776991.583568:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000020:00000040:3.0:1775776991.583572:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000400:00000200:2.0:1775776991.583574:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1775776991.583575:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776991.583578:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776991.583580:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776991.583581:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1775776991.583584:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 02000000:00000001:3.0:1775776991.583584:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776991.583586:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776991.583587:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776991.583591:0:301888:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:3.0:1775776991.583592:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000200:2.0:1775776991.583597:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776991.583598:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776991.583606:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776991.583611:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.583618:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776991.583618:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776991.583622:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776991.583630:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583631:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.583635:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776991.583639:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776991.583644:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000100:00000001:3.0:1775776991.583645:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776991.583650:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775776991.583651:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18717) 00000100:00000001:2.0:1775776991.583653:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313379336 00000100:00000001:2.0:1775776991.583657:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313380598 00000100:00000001:2.0:1775776991.583659:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313379336 00000100:00000001:2.0:1775776991.583661:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313380598 00000100:00000001:2.0:1775776991.583664:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776991.583666:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.583668:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776991.583672:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776991/real 1775776991] req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776991.583681:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776991.583683:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776991.583684:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776991.583687:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776991.583688:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776991.583691:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776991.583694:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776991.583702:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583708:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776991.583709:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.583711:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.583712:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776991.583715:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583722:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776991.583725:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583732:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776991.583734:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.583736:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.583737:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.583739:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776991.583740:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776991.583743:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776991.583746:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776991.583748:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776991.583751:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776991.583759:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820288:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775776991.583762:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.583764:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.583767:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dece00 x1862033099820288/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583773:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776991.583775:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776991.583777:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776991.583796:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776991.583797:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776991.583799:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775776991.583801:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776991.583803:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158c00. 02000000:00000001:2.0:1775776991.583805:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776991.583806:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776991.583808:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dece00. 00000100:00000001:2.0:1775776991.583810:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776991.583811:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.583813:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.583817:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776991.583819:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.583820:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776991.583822:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776991.583823:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776991.583827:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583833:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776991.583835:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776991.583837:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776991.583839:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776991.583840:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776991.583843:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820416:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776991.583846:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776991.583848:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776991.583849:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776991.583851:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776991.583854:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776991.583856:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776991.583860:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776991.583862:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776991.583865:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776991.583868:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099820416, portal 4 00000100:00000001:2.0:1775776991.583870:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776991.583872:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751265152 : -114841958286464 : ffff978d45defb80) 00000100:00000040:2.0:1775776991.583877:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583883:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776991.583886:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776991.583888:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776991.583890:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099820416, offset 0 00000400:00000200:2.0:1775776991.583894:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776991.583897:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776991.583900:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776991.583902:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776991.583905:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776991.583907:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776991.583909:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776991.583912:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4640:4640:256:4294967295] 192.168.201.139@tcp LPNI seq info [4640:4640:8:4294967295] 00000400:00000200:2.0:1775776991.583916:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776991.583923:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776991.583927:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776991.583929:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776991.583933:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776991.583935:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776991.583940:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776991.583942:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1775776991.583945:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776991.583947:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776991.583949:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776991.583953:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776991.583956:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776991.583957:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776991.583960:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776991.583963:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583971:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.583973:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.583980:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.583981:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776991.583983:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776991.583986:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775776991.583989:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776991.583991:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313379346 00000100:00000001:2.0:1775776991.583993:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313380598 00000100:00000001:2.0:1775776991.583995:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313379346 00000100:00000001:2.0:1775776991.583997:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313380598 00000100:00000001:2.0:1775776991.583999:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776991.584001:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.584002:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776991.584005:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776991/real 1775776991] req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776991.584012:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776991.584014:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776991.584015:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776991.584017:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776991.584018:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776991.584020:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776991.584023:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776991.584037:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.584043:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776991.584044:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.584046:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.584047:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776991.584050:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.584056:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776991.584059:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.584066:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776991.584069:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.584070:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.584071:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.584073:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776991.584075:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776991.584077:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776991.584080:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776991.584081:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776991.584085:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776991.584092:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820416:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776991.584096:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776991.584099:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.584112:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776991.584113:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776991.584115:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776991.584117:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776991.584119:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776991.584121:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820544:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776991.584124:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776991.584125:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776991.584126:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776991.584128:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775776991.584131:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776991.584133:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776991.584136:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775776991.584138:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775776991.584140:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775776991.584143:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099820544, portal 4 00000100:00000001:2.0:1775776991.584144:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776991.584146:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257088 : -114841958294528 : ffff978d45dedc00) 00000100:00000040:2.0:1775776991.584151:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.584157:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775776991.584160:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776991.584162:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775776991.584164:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099820544, offset 0 00000400:00000200:2.0:1775776991.584167:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775776991.584170:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776991.584173:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775776991.584175:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776991.584178:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775776991.584180:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776991.584182:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776991.584185:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4641:4641:256:4294967295] 192.168.201.139@tcp LPNI seq info [4641:4641:8:4294967295] 00000400:00000200:2.0:1775776991.584189:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775776991.584196:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775776991.584200:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776991.584203:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775776991.584207:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775776991.584209:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1775776991.584213:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775776991.584216:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1775776991.584220:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775776991.584221:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775776991.584224:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775776991.584227:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776991.584230:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775776991.584232:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775776991.584234:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776991.585299:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.585315:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.585322:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777016 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.585329:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.585331:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775776991.585340:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776991.585344:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775776991.585349:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776991.585351:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313379338 00000100:00000001:2.0:1775776991.585354:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313380598 00000100:00000001:2.0:1775776991.585357:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313379338 00000100:00000001:2.0:1775776991.585359:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313380598 00000100:00000001:2.0:1775776991.585361:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776991.585364:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.585366:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776991.585370:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776991/real 1775776991] req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776991.585459:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776991.585460:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776991.585463:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776991.585466:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776991.585468:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776991.585471:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776991.585474:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776991.585484:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.585490:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776991.585492:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.585494:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.585495:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776991.585498:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.585506:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776991.585509:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.585517:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776991.585519:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.585521:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776991.585522:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.585524:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776991.585526:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776991.585530:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776991.585534:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776991.585536:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776991.585540:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776991.585548:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820544:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775776991.585552:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.585554:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.585557:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45defb80 x1862033099820416/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.585564:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776991.585566:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776991.585569:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504c00. 02000000:00000001:2.0:1775776991.585572:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776991.585574:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776991.585575:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775776991.585578:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776991.585579:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:2.0:1775776991.585583:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776991.585584:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776991.585586:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45defb80. 00000100:00000001:2.0:1775776991.585589:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776991.585590:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.585591:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776991.585594:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedc00 x1862033099820544/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777016 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776991.585600:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776991.585602:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776991.585603:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776991.585605:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776991.585606:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776991.585607:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775776991.585608:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776991.585610:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:2.0:1775776991.585612:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776991.585613:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776991.585615:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedc00. 00000100:00000001:2.0:1775776991.585617:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776991.585618:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.585620:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776991.585625:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776991.585626:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776991.585629:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776991.585630:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776992.284126:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776992.284141:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776992.284143:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776992.284152:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776992.284158:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776992.284164:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776992.284169:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776992.284173:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776992.284177:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776992.349476:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776992.349483:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776992.349489:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776992.349490:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776992.349494:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776992.349495:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776992.349498:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776992.349499:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776992.350126:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776992.350130:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776992.350133:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776992.604120:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776992.604127:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776992.604128:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776992.604134:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776992.604146:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776992.604150:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776992.604153:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776992.604155:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776992.604158:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776992.859976:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313380598, at 4313380608 00000100:00000010:0.0:1775776992.859991:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000040:1.0:1775776992.861883:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313380598, at 4313380610 00000100:00000010:1.0:1775776992.861895:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000001:3.0:1775776993.307883:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776993.307889:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776993.307891:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776993.307896:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776993.307900:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776993.307903:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776993.307906:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776993.307908:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776993.307910:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776993.372730:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776993.372736:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776993.372752:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776993.372769:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776993.372821:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776993.372823:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776993.374299:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776993.374303:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776993.374513:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776993.375354:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776993.375359:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776993.627944:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776993.627951:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776993.627953:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776993.627958:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776993.627962:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776993.627966:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776993.627968:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776993.627970:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776993.627973:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776994.335384:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776994.335391:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776994.335393:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776994.335398:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776994.335401:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776994.335405:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776994.335407:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776994.335410:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776994.335412:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776994.396362:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776994.396368:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776994.396374:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776994.396375:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776994.396379:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776994.396380:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776994.396384:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776994.396385:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776994.396502:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776994.396506:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776994.396509:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776994.652894:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776994.652901:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776994.652903:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776994.652909:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776994.652913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776994.652916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776994.652919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776994.652921:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776994.652924:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776995.356019:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776995.356026:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776995.356028:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776995.356034:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776995.356037:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776995.356041:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776995.356044:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776995.356046:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776995.356048:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776995.420456:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776995.420463:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776995.420469:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776995.420471:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776995.420474:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776995.420475:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776995.420478:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776995.420480:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776995.421473:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776995.421487:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776995.421491:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776995.675928:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776995.675934:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776995.675936:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776995.675942:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775776995.675946:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776995.675949:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776995.675952:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776995.675954:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776995.675956:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.380029:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776996.380038:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776996.380040:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776996.380046:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776996.380050:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776996.380054:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.380057:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776996.380059:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.380062:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776996.443957:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776996.443964:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776996.443969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776996.443970:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776996.443974:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776996.443975:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776996.443979:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776996.443980:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776996.444088:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776996.444094:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776996.444101:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776996.699940:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776996.699960:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776996.699963:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776996.699967:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776996.699971:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776996.699978:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18712 00000400:00000200:3.0:1775776996.699989:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.699995:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776996.699998:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776996.700001:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700005:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776996.700007:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700009:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776996.700013:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776996.700015:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775776996.700018:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776996.700020:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775776996.700023:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775776996.700025:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775776996.700028:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700030:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776996.700031:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700033:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776996.700036:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775776996.700039:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700041:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.700045:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776996.700049:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776996.700052:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.700054:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776996.700057:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776996.700067:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775776996.700071:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775776996.700075:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776996.700077:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.700080:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776996.700083:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776996.700087:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776996.700095:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775776996.700102:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776996.700111:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776996.700121:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776996.700125:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776996.700130:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.700167:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776996.700171:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776996.700174:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776996.700176:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776996.700177:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776996.700180:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776996.700182:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18712 00000400:00000200:3.0:1775776996.700186:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.700190:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776996.700193:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776996.700196:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700199:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776996.700202:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700204:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776996.700206:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776996.700207:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000100:00000001:2.0:1775776996.700208:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776996.700209:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776996.700213:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000100:00000001:2.0:1775776996.700214:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776996.700216:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000001:2.0:1775776996.700219:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775776996.700219:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775776996.700224:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700228:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776996.700229:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776996.700232:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700238:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776996.700242:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00000001:2.0:1775776996.700248:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776996.700249:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775776996.700253:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.700254:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.700257:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776996.700259:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776996.700262:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776996.700263:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776996.700266:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.700268:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.700271:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:2.0:1775776996.700273:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099820672:192.168.201.139@tcp:250:kworker.0 00000008:00100000:3.0:1775776996.700276:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1775776996.700280:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1775776996.700281:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.700284:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776996.700285:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1775776996.700288:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 02000000:00000001:2.0:1775776996.700289:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776996.700292:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775776996.700293:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776996.700297:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776996.700299:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.700300:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1775776996.700301:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507800. 02000000:00000001:3.0:1775776996.700302:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776996.700306:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.700306:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776996.700315:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 02000000:00000010:3.0:1775776996.700316:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000400:00000010:2.0:1775776996.700321:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000100:00000001:3.0:1775776996.700321:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776996.700329:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1775776996.700330:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000001:3.0:1775776996.700340:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1775776996.700341:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099820672, portal 25 00000100:00000001:3.0:1775776996.700344:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776996.700345:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776996.700348:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751263360 : -114841958288256 : ffff978d45def480) 00000100:00000040:3.0:1775776996.700351:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.700362:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776996.700365:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776996.700366:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776996.700371:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776996.700374:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776996.700377:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776996.700383:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776996.700384:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776996.700389:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18712 00000100:00000040:2.0:1775776996.700390:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775776996.700395:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000400:00000200:3.0:1775776996.700397:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000200:2.0:1775776996.700400:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099820672, offset 0 00000400:00000200:3.0:1775776996.700403:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776996.700406:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776996.700411:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:2.0:1775776996.700413:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700416:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776996.700419:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700422:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776996.700425:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:3.0:1775776996.700426:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776996.700429:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775776996.700432:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776996.700434:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000400:00000200:2.0:1775776996.700436:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000020:00000001:3.0:1775776996.700438:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775776996.700442:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:2.0:1775776996.700445:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776996.700447:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700450:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1775776996.700453:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00000040:3.0:1775776996.700454:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700457:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:2.0:1775776996.700461:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1775776996.700462:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000400:00000200:2.0:1775776996.700466:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775776996.700471:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4642:4642:256:4294967295] 192.168.201.139@tcp LPNI seq info [4642:4642:8:4294967295] 00000100:00080000:3.0:1775776996.700477:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000400:00000200:2.0:1775776996.700478:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776996.700483:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.700485:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776996.700488:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776996.700493:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776996.700497:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00000001:3.0:1775776996.700498:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776996.700501:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776996.700505:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776996.700509:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700509:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1775776996.700517:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000100:00000010:3.0:1775776996.700521:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000020:00000040:3.0:1775776996.700525:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000800:00000200:2.0:1775776996.700529:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000001:3.0:1775776996.700530:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776996.700533:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775776996.700535:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1775776996.700536:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776996.700538:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776996.700541:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775776996.700545:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 02000000:00000010:3.0:1775776996.700548:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000800:00000200:2.1:1775776996.700550:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1775776996.700552:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1775776996.700558:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00100000:3.0:1775776996.700561:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:2.0:1775776996.700562:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000100:00000001:3.0:1775776996.700572:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1775776996.700573:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776996.700574:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776996.700583:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776996.700586:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775776996.700596:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000100:00000001:3.0:1775776996.700597:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1775776996.700601:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00080000:3.0:1775776996.700602:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776996.700608:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775776996.700609:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775776996.700609:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776996.700613:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1775776996.700617:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.700617:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776996.700624:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18712 00000400:00000200:3.0:1775776996.700633:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776996.700637:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1775776996.700638:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776996.700642:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776996.700645:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776996.700648:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700653:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776996.700657:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775776996.700660:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.700661:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000100:00000001:2.0:1775776996.700663:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1775776996.700667:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776996.700668:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000100:00000001:2.0:1775776996.700670:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776996.700675:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775776996.700678:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313385717 00000020:00000001:3.0:1775776996.700681:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000010:2.0:1775776996.700683:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b8a0. 00000020:00000040:3.0:1775776996.700684:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775776996.700689:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775776996.700691:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000001:2.0:1775776996.700692:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1775776996.700696:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.700699:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776996.700702:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313385717 00000100:00000010:2.0:1775776996.700705:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b3c0. 00000100:00000001:2.0:1775776996.700708:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776996.700711:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776996.700714:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776996.700716:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775776996.700720:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776996/real 1775776996] req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775776996.700731:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775776996.700733:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775776996.700734:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775776996.700737:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775776996.700739:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775776996.700741:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775776996.700745:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775776996.700754:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776996.700760:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776996.700762:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776996.700764:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776996.700766:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775776996.700769:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776996.701867:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.701872:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776996.701875:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775776996.701878:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776996.701968:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.701971:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776996.701978:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776996.701980:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.701983:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776996.701986:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776996.701989:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.701990:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776996.701996:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775776996.702000:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775776996.702003:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776996.702005:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.702008:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776996.702010:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776996.702013:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776996.702019:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775776996.702025:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776996.702032:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776996.702042:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776996.702045:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776996.702049:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.702057:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776996.702061:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776996.702065:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775776996.702066:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776996.702068:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776996.702071:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776996.702074:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18712 00000400:00000200:3.0:1775776996.702080:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.702084:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776996.702086:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776996.702089:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.702091:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776996.702093:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.702095:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775776996.702097:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776996.702098:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775776996.702100:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776996.702102:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775776996.702104:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775776996.702106:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775776996.702109:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.702111:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776996.702112:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775776996.702114:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775776996.702116:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775776996.702118:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775776996.702121:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.702122:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776996.702126:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776996.702127:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.702129:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776996.702131:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775776996.702134:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.702136:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776996.702140:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775776996.702143:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775776996.702145:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776996.702146:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.702199:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776996.702201:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776996.702202:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776996.702207:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775776996.702210:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776996.702214:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776996.702222:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776996.702224:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776996.702227:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.702235:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776996.702240:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18722) 00000100:00000001:2.0:1775776996.715883:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776996.715906:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776996.715918:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775776996.715923:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776996.715926:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775776996.715927:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776996.715931:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775776996.715933:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775776996.715937:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775776996.715941:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775776996.715944:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775776996.715949:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776996.715958:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099820672:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775776996.715963:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776996.716046:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776996.716050:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def480 x1862033099820672/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776996.716059:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776996.716062:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776996.716067:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507800. 02000000:00000001:2.0:1775776996.716073:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776996.716075:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776996.716077:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775776996.716079:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776996.716081:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158400. 02000000:00000001:2.0:1775776996.716086:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776996.716088:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776996.716090:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def480. 00000100:00000001:2.0:1775776996.716095:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776996.716097:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776996.716099:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776996.716106:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776996.716108:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776996.716110:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776996.716113:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776996.716216:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776996.716223:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776996.716232:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776996.716234:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776996.716237:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776996.716240:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776996.716243:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776996.716246:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775776996.716250:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776996.716255:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776996.716257:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776996.716259:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776996.716269:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775776996.716271:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776996.716279:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775776996.716283:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775776996.716291:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775776996.716298:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099820800, portal 10 00000100:00000001:0.0:1775776996.716301:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776996.716302:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751262464 : -114841958289152 : ffff978d45def100) 00000100:00000040:0.0:1775776996.716310:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776996.716318:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776996.716321:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775776996.716324:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775776996.716327:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099820800, offset 0 00000400:00000200:0.0:1775776996.716333:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775776996.716341:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776996.716348:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775776996.716353:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776996.716357:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775776996.716362:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776996.716364:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775776996.716369:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4643:4643:256:4294967295] 192.168.201.139@tcp LPNI seq info [4643:4643:8:4294967295] 00000400:00000200:0.0:1775776996.716373:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775776996.716383:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775776996.716388:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776996.716393:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775776996.716399:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775776996.716402:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775776996.716408:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775776996.716411:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1775776996.716416:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775776996.716418:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775776996.716423:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776996.716428:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776996.716432:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775776996.716435:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775776996.716438:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776996.716442:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776996.716451:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776996.716454:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776996.716461:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776996.716463:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776996.716467:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776996.716471:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775776996.716474:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775776996.716476:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313384468 00000100:00000001:0.0:1775776996.716480:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313385717 00000100:00000001:0.0:1775776996.716483:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313384468 00000100:00000001:0.0:1775776996.716485:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313385717 00000100:00000001:0.0:1775776996.716487:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776996.716489:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776996.758708:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776996.758727:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776996/real 1775776996] req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776996.758746:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776996.758748:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776996.758752:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775776996.758759:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775776996.758762:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775776996.758766:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776996.758770:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776996.758948:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776996.758957:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776996.758960:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776996.758963:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776996.758965:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776996.758969:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776996.758976:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776996.758981:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776996.758989:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776996.758992:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776996.758994:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776996.758995:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776996.758998:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776996.759001:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776996.759004:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776996.759024:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775776996.759026:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776996.759030:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776996.759039:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820800:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776996.764294:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776996.764312:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.764325:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776996.764333:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.764337:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776996.764341:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776996.764344:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776996.764348:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776996.764353:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776996.764357:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776996.764360:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.764362:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776996.764372:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776996.764374:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776996.764383:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776996.764388:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775776996.764396:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776996.764403:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099820928, portal 10 00000100:00000001:3.0:1775776996.764406:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776996.764407:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751261568 : -114841958290048 : ffff978d45deed80) 00000100:00000040:3.0:1775776996.764415:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.764423:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776996.764425:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776996.764428:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776996.764432:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099820928, offset 0 00000400:00000200:3.0:1775776996.764439:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776996.764448:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.764457:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776996.764462:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776996.764467:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776996.764470:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776996.764473:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776996.764477:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4644:4644:256:4294967295] 192.168.201.139@tcp LPNI seq info [4644:4644:8:4294967295] 00000400:00000200:3.0:1775776996.764482:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.764492:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776996.764498:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776996.764503:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776996.764510:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776996.764513:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776996.764519:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776996.764522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775776996.764526:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776996.764529:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776996.764534:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776996.764539:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776996.764544:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776996.764546:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776996.764551:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776996.764555:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.764566:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776996.764569:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.764576:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.764577:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776996.764582:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776996.764587:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775776996.764590:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776996.764594:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313384496 00000100:00000001:3.0:1775776996.764597:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313385717 00000100:00000001:3.0:1775776996.764600:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313384496 00000100:00000001:3.0:1775776996.764602:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313385717 00000100:00000001:3.0:1775776996.764605:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776996.764607:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.764610:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776996.764614:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776996/real 1775776996] req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776996.764624:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776996.764626:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776996.764627:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775776996.764630:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776996.764631:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776996.764634:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776996.764637:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776996.764646:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.764653:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776996.764655:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.764657:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776996.764659:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776996.764663:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.764670:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776996.764674:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.764681:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776996.764684:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.764686:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776996.764687:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.764690:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776996.764692:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776996.764695:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776996.764700:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776996.764702:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776996.764706:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776996.764715:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099820928:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775776996.764719:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776996.764723:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.764729:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776996.764731:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.764733:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776996.764735:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776996.764737:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776996.764740:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776996.764743:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776996.764745:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776996.764747:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.764748:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776996.764751:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159400. 02000000:00000001:3.0:1775776996.764754:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776996.764757:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776996.764759:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775776996.764762:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776996.764766:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099821056, portal 4 00000100:00000001:3.0:1775776996.764767:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776996.764768:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751251712 : -114841958299904 : ffff978d45dec700) 00000100:00000040:3.0:1775776996.764774:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.766388:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776996.766394:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776996.766402:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776996.766409:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099821056, offset 0 00000400:00000200:3.0:1775776996.766415:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776996.766423:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.766430:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776996.766435:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776996.766438:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776996.766441:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776996.766444:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776996.766447:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4645:4645:256:4294967295] 192.168.201.139@tcp LPNI seq info [4645:4645:8:4294967295] 00000400:00000200:3.0:1775776996.766452:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.766460:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776996.766466:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776996.766470:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776996.766475:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776996.766479:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776996.766485:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776996.766488:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775776996.766492:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776996.766495:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776996.766499:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776996.766503:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776996.766508:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776996.766510:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776996.766514:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776996.766519:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.766529:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776996.766533:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.766546:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.766547:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776996.766551:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776996.766556:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775776996.766559:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776996.766562:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313384505 00000100:00000001:3.0:1775776996.766565:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313385717 00000100:00000001:3.0:1775776996.766568:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313384505 00000100:00000001:3.0:1775776996.766569:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313385717 00000100:00000001:3.0:1775776996.766572:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776996.766574:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.766577:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776996.766581:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776996/real 1775776996] req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776996.766590:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776996.766592:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776996.766594:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775776996.766597:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776996.766598:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776996.766601:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776996.766604:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776996.766614:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.766620:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776996.766622:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.766624:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776996.766626:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776996.766630:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.766637:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776996.766642:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.766649:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776996.766651:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.766653:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776996.766654:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.766656:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776996.766659:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776996.766662:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776996.766667:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776996.770519:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776996.770538:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776996.770552:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821056:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776996.770560:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776996.770564:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.770572:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776996.770574:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.770579:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776996.770583:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776996.770587:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776996.770590:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776996.770595:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776996.770599:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776996.770604:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776996.770607:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776996.770616:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159000. 02000000:00000001:3.0:1775776996.770619:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776996.770629:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775776996.770634:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775776996.770643:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775776996.770650:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099821184, portal 4 00000100:00000001:3.0:1775776996.770652:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776996.770654:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257088 : -114841958294528 : ffff978d45dedc00) 00000100:00000040:3.0:1775776996.770662:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.772483:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776996.772489:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775776996.772497:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775776996.772503:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099821184, offset 0 00000400:00000200:3.0:1775776996.772511:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775776996.772521:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.772529:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775776996.772534:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776996.772538:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775776996.772541:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776996.772544:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775776996.772548:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4646:4646:256:4294967295] 192.168.201.139@tcp LPNI seq info [4646:4646:8:4294967295] 00000400:00000200:3.0:1775776996.772553:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775776996.774276:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775776996.774284:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776996.774293:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775776996.774301:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775776996.774305:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775776996.774311:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775776996.774314:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1775776996.774319:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775776996.774322:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775776996.774329:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775776996.774333:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775776996.774340:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775776996.774343:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775776996.774348:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776996.774355:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.774368:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776996.774372:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777021 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.774965:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.774968:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776996.774976:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776996.774981:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775776996.774987:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775776996.774990:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313384496 00000100:00000001:3.0:1775776996.774994:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313385717 00000100:00000001:3.0:1775776996.775004:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313384496 00000100:00000001:3.0:1775776996.775007:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313385717 00000100:00000001:3.0:1775776996.775009:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776996.775012:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.775390:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775776996.775398:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776996/real 1775776996] req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775776996.775411:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775776996.775413:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775776996.775416:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775776996.775421:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775776996.775422:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775776996.775426:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775776996.775430:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775776996.775439:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.775447:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775776996.775449:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.775451:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776996.775453:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775776996.775458:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.775465:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776996.775470:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.775478:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776996.775482:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.775484:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775776996.775485:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.775488:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775776996.775491:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775776996.775494:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775776996.775499:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775776996.775502:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775776996.775506:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776996.775514:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821184:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775776996.778340:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.778344:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776996.778355:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def100 x1862033099820800/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.778364:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776996.778368:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776996.778373:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759dc00. 02000000:00000001:3.0:1775776996.778379:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776996.778382:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776996.778384:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775776996.778386:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776996.778389:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775776996.778392:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776996.778394:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776996.778396:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def100. 00000100:00000001:3.0:1775776996.778400:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776996.778401:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.778404:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776996.778408:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deed80 x1862033099820928/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.778414:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776996.778416:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776996.778417:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115a400. 02000000:00000001:3.0:1775776996.778420:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776996.778421:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776996.778422:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775776996.778424:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776996.778426:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:3.0:1775776996.778428:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776996.778429:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776996.778430:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deed80. 00000100:00000001:3.0:1775776996.778433:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776996.778434:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.778436:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776996.778440:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec700 x1862033099821056/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.778447:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776996.778449:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776996.778451:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159400. 02000000:00000001:3.0:1775776996.778453:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776996.778455:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776996.778456:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775776996.778457:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776996.778459:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775776996.778461:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776996.778462:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776996.778463:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec700. 00000100:00000001:3.0:1775776996.778465:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776996.778466:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.778468:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776996.778471:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedc00 x1862033099821184/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777021 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776996.778478:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775776996.778479:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775776996.778481:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159000. 02000000:00000001:3.0:1775776996.778483:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775776996.778484:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775776996.778485:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775776996.778487:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775776996.778488:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775776996.778490:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775776996.778491:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775776996.778492:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedc00. 00000100:00000001:3.0:1775776996.778494:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775776996.778495:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.778498:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776996.778506:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776996.778509:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776996.778511:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776996.778513:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776996.898211:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776996.898219:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776996.898221:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776996.898229:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776996.898233:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776996.898237:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776996.898240:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776996.898244:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776996.898247:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776997.404281:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776997.404292:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776997.404293:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776997.404299:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776997.404304:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776997.404308:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776997.404311:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776997.404313:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776997.404316:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776997.467944:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776997.467950:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776997.467973:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776997.467974:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776997.467978:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776997.467979:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776997.467983:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776997.467984:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776997.468101:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776997.468104:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776997.468107:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776997.915988:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776997.915994:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776997.915996:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776997.916002:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776997.916006:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776997.916009:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776997.916012:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776997.916015:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776997.916018:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776997.980105:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313385717, at 4313385728 00000100:00000010:1.0:1775776997.980120:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b3c0. 00000100:00000040:0.0:1775776997.980567:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313385717, at 4313385728 00000100:00000010:0.0:1775776997.980574:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000001:3.0:1775776998.428250:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776998.428256:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776998.428258:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776998.428264:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776998.428267:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776998.428271:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776998.428273:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776998.428276:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776998.428278:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776998.492444:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776998.492451:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776998.492469:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776998.492470:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776998.492474:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776998.492475:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776998.492478:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776998.492480:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776998.500999:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776998.501005:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776998.501010:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776998.939956:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776998.939963:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776998.939964:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776998.939970:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776998.939974:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776998.939977:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776998.939980:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776998.939983:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776998.939986:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776999.455462:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776999.455468:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776999.455470:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776999.455475:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775776999.455479:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776999.455483:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776999.455485:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776999.455487:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776999.455490:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776999.515916:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776999.515925:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775776999.515931:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775776999.515934:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776999.515940:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775776999.515942:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776999.515944:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776999.515946:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776999.515950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776999.515951:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776999.515954:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776999.515955:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775776999.515959:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775776999.515960:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776999.516095:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776999.516130:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776999.516134:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776999.963951:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776999.963960:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776999.963962:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776999.963969:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775776999.963973:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776999.963977:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776999.963979:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776999.963982:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776999.963984:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777000.475977:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777000.475989:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777000.475991:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777000.476000:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777000.476005:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777000.476011:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777000.476014:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777000.476017:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777000.476019:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777000.539910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777000.539916:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777000.539921:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777000.539922:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777000.539925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777000.539926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777000.539929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777000.539930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777000.540297:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777000.540300:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777000.540303:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777000.987950:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777000.987956:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777000.987957:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777000.987963:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777000.987967:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777000.987971:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777000.987974:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777000.987976:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777000.987979:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.500003:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777001.500010:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777001.500012:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777001.500017:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777001.500021:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777001.500026:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.500029:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777001.500032:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.500034:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566233:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777001.566239:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566245:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777001.566248:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777001.566251:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777001.566254:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566256:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777001.566259:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566260:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566263:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566266:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777001.566267:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777001.566269:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777001.566270:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566272:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777001.566273:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566275:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566276:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566280:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777001.566281:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777001.566282:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777001.566283:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566285:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777001.566286:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566288:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777001.566289:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777001.566659:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777001.566662:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777001.566665:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775777001.819927:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777001.819938:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775777001.819941:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777001.819946:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777001.819951:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777001.819958:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18717 00000400:00000200:1.0:1775777001.819968:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777001.819974:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777001.819977:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777001.819980:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.819984:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.819986:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.819988:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777001.819993:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777001.819994:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775777001.819997:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777001.820000:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775777001.820003:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775777001.820010:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775777001.820013:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.820015:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.820016:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.820018:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777001.820021:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775777001.820024:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777001.820027:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777001.820030:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777001.820043:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777001.820046:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.820049:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775777001.820051:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777001.820200:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775777001.820204:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775777001.820208:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777001.820210:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.820213:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777001.820216:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777001.820221:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777001.820229:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775777001.820235:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777001.820245:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777001.820255:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777001.820260:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777001.820264:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.820312:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775777001.820316:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777001.820318:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777001.820324:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.820332:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777001.820333:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.820337:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777001.820339:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777001.820342:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777001.820345:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099821312:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775777001.820349:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777001.820352:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777001.820355:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.820357:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777001.820362:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775777001.820364:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777001.820369:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777001.820373:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777001.820380:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777001.820388:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099821312, portal 25 00000100:00000001:3.0:1775777001.820390:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777001.820392:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609088 : -114841421942528 : ffff978d65d6f100) 00000100:00000040:3.0:1775777001.820397:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.820405:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777001.820407:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777001.820410:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777001.820413:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099821312, offset 0 00000400:00000200:3.0:1775777001.820420:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777001.820427:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.820433:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777001.820438:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777001.820442:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777001.820445:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777001.820448:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777001.820452:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4647:4647:256:4294967295] 192.168.201.139@tcp LPNI seq info [4647:4647:8:4294967295] 00000400:00000200:3.0:1775777001.820457:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.820465:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777001.820471:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.820475:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777001.820481:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777001.820484:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775777001.820489:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777001.820492:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775777001.820496:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.820498:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777001.820504:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777001.820508:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777001.820512:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777001.820514:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777001.820518:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777001.820521:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777001.820698:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777001.820702:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777001.820705:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775777001.820707:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777001.820708:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777001.820711:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777001.820713:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18717 00000400:00000200:1.0:1775777001.820717:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777001.820720:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777001.820722:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777001.820725:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.820727:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.820729:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.820731:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777001.820733:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777001.820734:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775777001.820735:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777001.820737:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775777001.820739:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775777001.820740:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775777001.820743:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.820745:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.820746:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.820748:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777001.820749:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775777001.820752:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777001.820754:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777001.820756:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777001.820761:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777001.820763:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.820765:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777001.820768:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775777001.820771:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.820773:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777001.820776:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775777001.820806:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775777001.820809:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777001.820810:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.820812:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777001.820814:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777001.820816:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777001.820820:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775777001.820823:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777001.820829:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777001.820835:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777001.820837:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777001.820840:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.821649:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.821656:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.821664:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.821666:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777001.821741:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777001.821746:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777001.821749:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313390838 00000100:00000010:3.0:1775777001.821756:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503446c0. 00000100:00000001:3.0:1775777001.821763:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775777001.821772:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313390838 00000100:00000010:3.0:1775777001.821790:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d503449c0. 00000100:00000001:3.0:1775777001.821793:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775777001.821797:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777001.821799:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.821802:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777001.821807:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777001/real 1775777001] req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777001.821818:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777001.821819:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777001.821822:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777001.821825:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777001.821827:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777001.821830:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777001.821833:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777001.821843:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.821850:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777001.821851:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.821853:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.821855:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777001.821859:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.821866:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777001.821871:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.821879:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777001.821882:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.821883:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.821885:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.821887:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777001.821889:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777001.821892:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777001.821897:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777001.821899:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777001.821903:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777001.821912:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099821312:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775777001.821917:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.821918:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.821921:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f100 x1862033099821312/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.821928:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777001.821931:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777001.821933:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775777001.821936:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777001.821938:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777001.821940:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775777001.821942:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777001.821944:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b000. 02000000:00000001:3.0:1775777001.821948:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777001.821950:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777001.821951:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f100. 00000100:00000001:3.0:1775777001.821954:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777001.821956:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.821958:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.821966:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777001.821967:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.821970:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775777001.821972:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777001.821973:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777001.821976:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.821984:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777001.821985:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.821988:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777001.821990:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777001.821992:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777001.821995:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821440:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775777001.821999:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777001.822001:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777001.822002:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.822004:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777001.822007:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775777001.822009:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777001.822013:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777001.822016:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777001.822020:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777001.822023:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099821440, portal 10 00000100:00000001:3.0:1775777001.822025:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777001.822027:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287598336 : -114841421953280 : ffff978d65d6c700) 00000100:00000040:3.0:1775777001.822033:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.822040:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777001.822042:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777001.822044:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777001.822048:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099821440, offset 0 00000400:00000200:3.0:1775777001.822052:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822162:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777001.822294:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777001.822300:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775777001.822302:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777001.822304:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777001.822306:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777001.822310:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18717 00000400:00000200:1.0:1775777001.822315:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777001.822320:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777001.822323:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777001.822325:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822329:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.822331:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822334:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777001.822337:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777001.822338:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775777001.822340:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777001.822342:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775777001.822344:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775777001.822347:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775777001.822350:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822352:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.822353:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822355:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777001.822358:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775777001.822361:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822364:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777001.822366:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777001.822369:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777001.822371:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822374:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777001.822377:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775777001.822379:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822380:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777001.822385:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775777001.822387:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775777001.822390:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777001.822391:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822394:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777001.822396:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777001.822400:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777001.822404:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775777001.822408:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777001.822413:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777001.822421:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777001.822424:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777001.822428:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777001.822436:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777001.822439:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777001.822441:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775777001.822443:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777001.822444:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777001.822447:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777001.822449:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18717 00000400:00000200:1.0:1775777001.822452:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777001.822455:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777001.822457:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777001.822460:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822462:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.822464:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822466:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777001.822468:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777001.822469:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775777001.822471:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777001.822472:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775777001.822475:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775777001.822477:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775777001.822480:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822482:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.822483:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822485:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777001.822488:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775777001.822490:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822493:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777001.822494:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777001.822497:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777001.822499:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822501:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777001.822503:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775777001.822809:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822812:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777001.822816:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775777001.822819:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775777001.822822:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777001.822823:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822824:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777001.822826:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777001.822828:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777001.822831:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775777001.822835:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777001.822839:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777001.822846:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777001.822848:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777001.822851:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777001.822858:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777001.822861:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777001.822864:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775777001.822865:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777001.822867:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777001.822869:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777001.822871:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18717 00000400:00000200:1.0:1775777001.822874:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777001.822877:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777001.822880:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777001.822882:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822884:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.822886:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822887:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777001.822889:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777001.822890:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775777001.822892:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777001.822894:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775777001.822896:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775777001.822897:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775777001.822900:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822902:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777001.822903:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822905:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777001.822907:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775777001.822910:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777001.822912:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777001.822913:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777001.822916:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777001.822918:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822920:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777001.822922:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775777001.822924:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822925:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777001.822928:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775777001.822930:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775777001.822932:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777001.822933:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777001.822935:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777001.822936:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777001.822937:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777001.822940:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775777001.822943:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777001.822946:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777001.822952:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777001.822954:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777001.822957:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777001.822963:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777001.822967:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18727) 00000400:00000200:3.0:1775777001.823031:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.823040:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777001.823044:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777001.823048:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777001.823051:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777001.823053:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777001.823057:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4648:4648:256:4294967295] 192.168.201.139@tcp LPNI seq info [4648:4648:8:4294967295] 00000400:00000200:3.0:1775777001.823075:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.823083:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777001.823089:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.823094:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777001.823100:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777001.823104:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775777001.823109:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777001.823112:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775777001.823116:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.823118:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777001.823123:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777001.823127:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777001.823131:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777001.823133:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777001.823136:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777001.823139:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823148:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.823150:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823156:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.823158:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777001.823161:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777001.823287:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775777001.823292:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777001.823294:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313389575 00000100:00000001:3.0:1775777001.823297:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313390838 00000100:00000001:3.0:1775777001.823305:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313389575 00000100:00000001:3.0:1775777001.823308:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313390838 00000100:00000001:3.0:1775777001.823311:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777001.823315:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.823317:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777001.823323:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777001/real 1775777001] req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777001.823332:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777001.823333:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777001.823336:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777001.823338:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777001.823340:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777001.823342:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777001.823345:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777001.823352:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823358:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777001.823359:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.823361:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.823362:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777001.823365:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823371:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777001.823376:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823383:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777001.823385:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.823386:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.823388:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.823389:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777001.823393:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777001.823396:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777001.823400:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777001.823402:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777001.823407:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777001.823414:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821440:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775777001.823419:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.823420:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.823422:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c700 x1862033099821440/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823428:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777001.823430:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777001.823433:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775777001.823436:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777001.823437:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777001.823438:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775777001.823440:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777001.823442:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39000. 02000000:00000001:3.0:1775777001.823444:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777001.823447:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777001.823448:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c700. 00000100:00000001:3.0:1775777001.823452:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777001.823453:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.823455:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.823463:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777001.823464:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.823466:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775777001.823468:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777001.823469:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777001.823473:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823479:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777001.823481:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.823484:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777001.823486:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777001.823488:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777001.823491:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821568:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775777001.823494:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777001.823496:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777001.823498:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.823500:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777001.823504:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775777001.823507:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777001.823510:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777001.823514:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777001.823519:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777001.823524:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099821568, portal 10 00000100:00000001:3.0:1775777001.823526:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777001.823527:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287597440 : -114841421954176 : ffff978d65d6c380) 00000100:00000040:3.0:1775777001.823533:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823540:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777001.823542:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777001.823544:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777001.823834:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099821568, offset 0 00000400:00000200:3.0:1775777001.823844:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777001.823849:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.823852:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777001.823855:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777001.823857:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777001.823860:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777001.823862:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777001.823865:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4649:4649:256:4294967295] 192.168.201.139@tcp LPNI seq info [4649:4649:8:4294967295] 00000400:00000200:3.0:1775777001.823868:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.823875:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777001.823880:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.823884:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777001.823888:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777001.823890:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775777001.823894:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777001.823896:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775777001.823900:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.823901:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777001.823905:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777001.823908:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777001.823911:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777001.823913:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777001.823915:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777001.823919:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823926:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.823928:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823934:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.823935:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777001.823938:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777001.823941:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775777001.823944:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777001.823946:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313389577 00000100:00000001:3.0:1775777001.823948:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313390838 00000100:00000001:3.0:1775777001.823950:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313389577 00000100:00000001:3.0:1775777001.823952:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313390838 00000100:00000001:3.0:1775777001.823954:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777001.823957:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.823958:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777001.823961:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777001/real 1775777001] req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777001.823970:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777001.823971:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777001.823972:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777001.823974:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777001.823975:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777001.823977:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777001.823979:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777001.823986:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.823991:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777001.823992:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.823994:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.823995:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777001.823998:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824003:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777001.824007:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824012:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777001.824014:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.824016:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.824017:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.824019:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777001.824020:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777001.824022:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777001.824025:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777001.824028:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777001.824032:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777001.824039:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821568:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775777001.824043:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777001.824046:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824051:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777001.824053:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.824055:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777001.824056:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777001.824069:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777001.824072:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821696:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775777001.824075:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777001.824077:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777001.824079:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.824080:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777001.824084:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159000. 02000000:00000001:3.0:1775777001.824086:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777001.824089:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777001.824092:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777001.824095:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777001.824097:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099821696, portal 4 00000100:00000001:3.0:1775777001.824099:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777001.824100:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287603712 : -114841421947904 : ffff978d65d6dc00) 00000100:00000040:3.0:1775777001.824107:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824112:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777001.824114:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777001.824115:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777001.824118:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099821696, offset 0 00000400:00000200:3.0:1775777001.824121:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777001.824125:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.824128:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777001.824130:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777001.824132:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777001.824134:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777001.824136:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777001.824139:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4650:4650:256:4294967295] 192.168.201.139@tcp LPNI seq info [4650:4650:8:4294967295] 00000400:00000200:3.0:1775777001.824142:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.824148:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777001.824151:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.824154:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777001.824158:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777001.824160:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775777001.824300:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777001.824304:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775777001.824309:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.824311:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777001.824314:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777001.824316:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777001.824319:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777001.824321:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777001.824323:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777001.824327:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824335:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.824338:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824344:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.824345:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777001.824348:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777001.824351:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775777001.824355:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777001.824357:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313389586 00000100:00000001:3.0:1775777001.824360:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313390838 00000100:00000001:3.0:1775777001.824362:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313389586 00000100:00000001:3.0:1775777001.824364:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313390838 00000100:00000001:3.0:1775777001.824366:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777001.824368:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.824369:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777001.824372:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777001/real 1775777001] req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777001.824378:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777001.824379:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777001.824381:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777001.824382:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777001.824384:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777001.824385:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777001.824388:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777001.824395:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824400:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777001.824401:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.824403:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.824404:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777001.824407:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824413:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777001.824415:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824421:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777001.824423:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.824425:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.824426:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.824427:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777001.824429:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777001.824432:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777001.824435:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777001.824437:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777001.824440:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777001.824446:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821696:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775777001.824449:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777001.824452:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824457:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777001.824459:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.824460:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777001.824462:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777001.824464:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777001.824466:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821824:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775777001.824470:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777001.824471:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777001.824473:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777001.824474:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777001.824478:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775777001.824479:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777001.824482:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777001.824484:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777001.824487:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777001.824490:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099821824, portal 4 00000100:00000001:3.0:1775777001.824492:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777001.824493:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287611776 : -114841421939840 : ffff978d65d6fb80) 00000100:00000040:3.0:1775777001.824498:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.824503:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777001.824505:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777001.824507:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777001.824509:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099821824, offset 0 00000400:00000200:3.0:1775777001.824512:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777001.824516:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.824520:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777001.824523:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777001.824526:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777001.824528:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777001.824530:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777001.824533:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4651:4651:256:4294967295] 192.168.201.139@tcp LPNI seq info [4651:4651:8:4294967295] 00000400:00000200:3.0:1775777001.824536:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777001.824541:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777001.824545:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.828103:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777001.828126:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777001.828130:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775777001.828136:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777001.828139:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1775777001.828142:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777001.828145:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777001.828153:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777001.828159:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777001.828305:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777001.828309:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777001.828315:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777001.828324:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.828342:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.828345:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777026 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.828352:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.828354:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777001.828362:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777001.828367:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775777001.828373:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777001.828376:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313389581 00000100:00000001:3.0:1775777001.828380:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313390838 00000100:00000001:3.0:1775777001.828382:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313389581 00000100:00000001:3.0:1775777001.828384:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313390838 00000100:00000001:3.0:1775777001.828386:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777001.828389:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.828392:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777001.828395:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777001/real 1775777001] req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777001.828406:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777001.828408:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777001.828410:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777001.828413:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777001.828415:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777001.828418:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777001.828420:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777001.828428:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.828435:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777001.828437:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.828439:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.828441:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777001.828445:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.828451:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777001.828455:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.828462:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777001.828465:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.828466:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777001.828467:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.828470:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777001.828472:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777001.828475:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777001.828480:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777001.828483:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777001.828486:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777001.828494:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099821824:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775777001.828499:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.828501:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.828504:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c380 x1862033099821568/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.828510:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777001.828513:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777001.828517:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775777001.828521:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777001.828523:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777001.828525:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775777001.828528:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777001.828530:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3bc00. 02000000:00000001:3.0:1775777001.828534:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777001.828535:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777001.828537:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c380. 00000100:00000001:3.0:1775777001.828541:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777001.828542:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.828544:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.828559:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6dc00 x1862033099821696/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.828565:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777001.828567:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777001.828568:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159000. 02000000:00000001:3.0:1775777001.828570:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777001.828571:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777001.828573:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775777001.828574:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777001.828575:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b800. 02000000:00000001:3.0:1775777001.828577:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777001.828578:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777001.828579:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6dc00. 00000100:00000001:3.0:1775777001.828730:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777001.828732:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.828733:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777001.828737:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6fb80 x1862033099821824/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777026 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777001.828743:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777001.828744:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777001.828746:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775777001.828748:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777001.828749:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777001.828750:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775777001.828751:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777001.828753:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a400. 02000000:00000001:3.0:1775777001.828755:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777001.828757:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777001.828758:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6fb80. 00000100:00000001:3.0:1775777001.828760:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777001.828761:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.828763:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777001.828770:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777001.828772:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777001.828775:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775777001.828776:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777002.012306:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777002.012313:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777002.012315:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777002.012320:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777002.012324:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777002.012328:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777002.012330:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777002.012333:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777002.012336:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777002.524680:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777002.524704:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777002.524707:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777002.524713:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777002.524717:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777002.524721:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777002.524724:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777002.524727:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777002.524729:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777002.587969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777002.587990:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777002.588010:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777002.588012:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777002.588018:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777002.588019:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777002.588022:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777002.588023:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777002.589166:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777002.589172:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777002.589177:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777003.035895:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777003.035902:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777003.035904:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777003.035909:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777003.035913:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777003.035916:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777003.035919:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777003.035922:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777003.035925:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777003.100114:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313390838, at 4313390848 00000100:00000010:1.0:1775777003.100130:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503449c0. 00000100:00000040:0.0:1775777003.102225:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313390838, at 4313390850 00000100:00000010:0.0:1775777003.102237:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d503446c0. 00000100:00000001:3.0:1775777003.547935:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777003.547942:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777003.547944:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777003.547949:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777003.547952:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777003.547956:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777003.547959:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777003.547961:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777003.547963:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777003.611905:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777003.611911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777003.611918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777003.611919:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777003.611922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777003.611923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777003.611926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777003.611928:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777003.614649:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777003.614655:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777003.614728:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777004.059930:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777004.059936:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777004.059938:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777004.059943:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777004.059946:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777004.059950:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777004.059952:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777004.059955:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777004.059958:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777004.572184:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777004.572191:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777004.572193:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777004.572210:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777004.572214:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777004.572218:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777004.572220:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777004.572222:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777004.572225:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777004.635975:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777004.635981:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777004.635988:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777004.635989:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777004.635992:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777004.635994:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777004.635997:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777004.635998:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777004.636105:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777004.636109:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777004.636113:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777005.085667:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777005.085674:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777005.085676:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777005.085689:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777005.085694:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777005.085697:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777005.085700:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777005.085702:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777005.085711:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777005.596361:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777005.596368:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777005.596369:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777005.596375:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777005.596379:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777005.596382:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777005.596385:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777005.596388:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777005.596390:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777005.661733:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777005.661740:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777005.661745:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777005.661747:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777005.661750:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777005.661751:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777005.661754:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777005.661755:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777005.663392:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777005.663397:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777005.663401:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777006.108001:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777006.108008:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777006.108009:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777006.108015:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777006.108019:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777006.108022:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777006.108025:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777006.108027:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777006.108029:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.619956:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777006.619962:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777006.619964:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777006.619969:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777006.619973:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777006.619984:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777006.619988:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777006.619990:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777006.619993:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777006.683960:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777006.683966:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777006.683972:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777006.684003:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777006.684008:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777006.684009:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777006.684012:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777006.684013:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777006.684110:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777006.684114:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777006.684116:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775777006.940283:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777006.940297:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775777006.940300:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777006.940305:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777006.940311:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777006.940319:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18722 00000400:00000200:3.0:1775777006.940333:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777006.940341:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777006.940344:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777006.940348:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940352:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.940354:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940356:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777006.940361:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777006.940372:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775777006.940376:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777006.940379:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775777006.940381:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775777006.940384:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775777006.940388:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940390:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.940392:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940394:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777006.940397:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775777006.940400:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940402:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777006.940405:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777006.940409:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777006.940412:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940415:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775777006.940418:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777006.940430:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775777006.940434:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775777006.940439:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777006.940441:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940445:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777006.940448:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777006.940453:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777006.940463:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775777006.940471:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777006.940482:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777006.940493:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777006.940498:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777006.940503:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777006.940530:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777006.940534:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777006.940538:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775777006.940539:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777006.940541:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777006.940544:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777006.940547:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18722 00000400:00000200:3.0:1775777006.940550:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777006.940553:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777006.940555:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777006.940558:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940560:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.940562:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940564:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777006.940566:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777006.940567:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775777006.940569:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777006.940571:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775777006.940573:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775777006.940575:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775777006.940578:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940579:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.940581:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940583:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777006.940585:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775777006.940588:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940591:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777006.940593:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777006.940600:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777006.940602:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940604:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777006.940609:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777006.940611:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940613:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777006.940618:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775777006.940620:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775777006.940623:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777006.940624:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940625:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777006.940627:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777006.940628:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777006.940632:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775777006.940634:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777006.940639:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777006.940646:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777006.940648:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777006.940651:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777006.940658:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777006.940661:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777006.940665:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775777006.940667:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777006.940668:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777006.940670:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777006.940673:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18722 00000400:00000200:3.0:1775777006.940677:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777006.940679:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777006.940681:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777006.940684:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940686:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.940688:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940690:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777006.940692:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777006.940693:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775777006.940695:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777006.940696:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775777006.940698:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775777006.940700:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775777006.940703:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940705:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.940707:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940708:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777006.940711:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775777006.940713:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940715:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777006.940717:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777006.940719:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777006.940721:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940723:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777006.940725:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777006.940727:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940729:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777006.940732:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000020:00000040:3.0:1775777006.940733:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775777006.940736:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777006.940737:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940739:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777006.940740:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777006.940741:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777006.940744:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775777006.940747:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777006.940750:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777006.940757:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777006.940759:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777006.940763:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777006.940770:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777006.940772:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777006.940775:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775777006.940817:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777006.940819:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777006.940821:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777006.940824:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18722 00000400:00000200:3.0:1775777006.940828:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777006.940831:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777006.940833:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777006.940836:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940838:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.940839:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940842:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777006.940844:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777006.940845:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775777006.940846:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777006.940848:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775777006.940850:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775777006.940851:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775777006.940854:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940856:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.940858:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940860:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777006.940874:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775777006.940877:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777006.940879:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777006.940881:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777006.940884:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777006.940886:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940887:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777006.940890:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777006.940892:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.940894:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777006.940897:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775777006.940899:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775777006.941235:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777006.941236:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.941238:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777006.941240:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777006.941242:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777006.941246:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775777006.941250:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777006.941255:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777006.941263:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777006.941264:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777006.941267:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777006.941275:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777006.941277:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777006.941281:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775777006.941282:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777006.941283:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777006.941286:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777006.941288:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18722 00000400:00000200:3.0:1775777006.941294:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777006.941298:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777006.941300:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777006.941303:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.941305:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.941307:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.941309:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777006.941311:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777006.941312:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775777006.941313:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777006.941315:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775777006.941317:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775777006.941319:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775777006.941322:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.941324:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777006.941326:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777006.941328:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777006.941331:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775777006.941333:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777006.941336:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777006.941338:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777006.941341:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777006.941343:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.941345:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777006.941348:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777006.941350:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.941351:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777006.941355:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775777006.941358:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775777006.941370:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777006.941372:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777006.941374:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777006.941375:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777006.941376:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777006.941380:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775777006.941384:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777006.941388:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777006.941395:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777006.941396:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777006.941400:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777006.941406:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777006.941412:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18732) 00000100:00000001:0.0:1775777006.941941:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777006.941948:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777006.941952:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777006.941965:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.941988:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777006.941991:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.941995:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777006.941998:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777006.942001:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777006.942005:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099821952:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777006.942009:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777006.942014:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777006.942016:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.942019:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777006.942029:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777006.942031:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777006.942039:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777006.942044:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777006.942051:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777006.942058:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099821952, portal 25 00000100:00000001:0.0:1775777006.942060:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777006.942062:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257088 : -114841958294528 : ffff978d45dedc00) 00000100:00000040:0.0:1775777006.942068:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942076:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777006.942078:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777006.942081:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777006.942083:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099821952, offset 0 00000400:00000200:0.0:1775777006.942115:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777006.942124:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.942132:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777006.942136:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777006.942140:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777006.942143:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777006.942145:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777006.942149:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4652:4652:256:4294967295] 192.168.201.139@tcp LPNI seq info [4652:4652:8:4294967295] 00000400:00000200:0.0:1775777006.942153:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.942161:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777006.942183:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.942188:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777006.942200:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777006.942203:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775777006.942208:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777006.942211:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775777006.942215:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.942217:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777006.942224:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777006.942228:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777006.942232:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777006.942234:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777006.942238:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777006.942241:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942251:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.942254:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942260:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.942261:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777006.942266:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777006.942269:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777006.942272:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313395959 00000100:00000010:0.0:1775777006.942277:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777006.942283:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777006.942290:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313395959 00000100:00000010:0.0:1775777006.942302:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237120. 00000100:00000001:0.0:1775777006.942306:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777006.942309:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777006.942311:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.942314:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777006.942317:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777006/real 1775777006] req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777006.942327:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777006.942328:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777006.942330:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777006.942333:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777006.942334:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777006.942336:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777006.942339:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777006.942353:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942360:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777006.942361:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.942364:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.942366:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777006.942369:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942376:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777006.942381:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942388:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777006.942390:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.942392:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.942393:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.942395:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777006.942397:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777006.942400:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777006.942404:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777006.942407:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777006.942410:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777006.942417:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099821952:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777006.942422:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777006.942425:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942431:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777006.942433:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.942435:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777006.942437:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777006.942439:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777006.942441:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822080:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777006.942444:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777006.942447:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777006.942448:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.942450:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777006.942453:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777006.942455:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777006.942458:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777006.942461:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777006.942464:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777006.942467:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099822080, portal 10 00000100:00000001:0.0:1775777006.942468:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777006.942470:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751251712 : -114841958299904 : ffff978d45dec700) 00000100:00000040:0.0:1775777006.942476:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942482:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777006.942484:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777006.942486:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777006.942488:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099822080, offset 0 00000400:00000200:0.0:1775777006.942491:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777006.942495:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.942498:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777006.942501:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777006.942503:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777006.942505:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777006.942507:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777006.942509:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4653:4653:256:4294967295] 192.168.201.139@tcp LPNI seq info [4653:4653:8:4294967295] 00000400:00000200:0.0:1775777006.942513:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.942518:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777006.942521:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.942524:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777006.942527:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777006.942529:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775777006.942533:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777006.942535:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775777006.942550:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.942552:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777006.942555:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777006.942559:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777006.942562:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777006.942564:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777006.942566:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777006.942569:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942575:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.942577:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942583:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.942584:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777006.942587:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777006.942591:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777006.942594:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777006.942596:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313394694 00000100:00000001:0.0:1775777006.942599:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313395959 00000100:00000001:0.0:1775777006.942601:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313394694 00000100:00000001:0.0:1775777006.942602:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313395959 00000100:00000001:0.0:1775777006.942604:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777006.942606:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.942618:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777006.942622:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777006/real 1775777006] req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777006.942631:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777006.942632:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777006.942633:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777006.942635:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777006.942637:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777006.942639:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777006.942641:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777006.942648:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942654:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777006.942655:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.942657:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.942658:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777006.942661:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942666:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777006.942669:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942674:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777006.942676:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.942678:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.942679:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.942680:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777006.942682:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777006.942684:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777006.942687:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777006.942689:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777006.942693:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777006.942700:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822080:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777006.942704:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777006.942707:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.942713:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777006.942945:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.942949:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777006.942951:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777006.942953:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777006.942957:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822208:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777006.942961:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777006.942963:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777006.942965:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.942967:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777006.942981:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777006.942983:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777006.942986:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777006.942988:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777006.942991:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777006.942994:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099822208, portal 10 00000100:00000001:0.0:1775777006.942996:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777006.942997:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751261568 : -114841958290048 : ffff978d45deed80) 00000100:00000040:0.0:1775777006.943003:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943009:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777006.943011:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777006.943012:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777006.943015:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099822208, offset 0 00000400:00000200:0.0:1775777006.943019:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777006.943023:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.943026:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777006.943028:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777006.943031:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777006.943033:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777006.943035:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777006.943037:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4654:4654:256:4294967295] 192.168.201.139@tcp LPNI seq info [4654:4654:8:4294967295] 00000400:00000200:0.0:1775777006.943040:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.943045:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777006.943049:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.943052:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777006.943056:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777006.943058:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775777006.943062:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777006.943065:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775777006.943068:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.943070:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777006.943073:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777006.943076:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777006.943079:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777006.943081:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777006.943083:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777006.943087:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943094:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.943097:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943102:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.943104:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777006.943107:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777006.943110:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777006.943113:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777006.943115:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313394696 00000100:00000001:0.0:1775777006.943117:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313395959 00000100:00000001:0.0:1775777006.943119:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313394696 00000100:00000001:0.0:1775777006.943121:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313395959 00000100:00000001:0.0:1775777006.943122:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777006.943124:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.943126:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777006.943129:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777006/real 1775777006] req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777006.943136:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777006.943137:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777006.943139:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777006.943141:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777006.943142:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777006.943144:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777006.943147:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777006.943153:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943159:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777006.943160:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.943162:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.943163:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777006.943166:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943195:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777006.943200:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943207:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777006.943210:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.943211:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.943212:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.943214:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777006.943215:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777006.943218:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777006.943221:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777006.943223:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777006.943227:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777006.943234:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822208:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777006.943237:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777006.943241:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943247:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777006.943248:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.943251:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777006.943252:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777006.943254:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777006.943257:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822336:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777006.943261:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777006.943263:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777006.943265:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.943266:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777006.943270:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775777006.943271:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777006.943275:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777006.943277:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777006.943281:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777006.943284:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099822336, portal 4 00000100:00000001:0.0:1775777006.943286:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777006.943288:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751262464 : -114841958289152 : ffff978d45def100) 00000100:00000040:0.0:1775777006.943293:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943299:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777006.943301:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777006.943303:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777006.943305:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099822336, offset 0 00000400:00000200:0.0:1775777006.943309:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777006.943313:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.943316:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777006.943318:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777006.943320:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777006.943322:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777006.943324:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777006.943327:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4655:4655:256:4294967295] 192.168.201.139@tcp LPNI seq info [4655:4655:8:4294967295] 00000400:00000200:0.0:1775777006.943330:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.943335:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777006.943339:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.943341:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777006.943346:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777006.943348:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775777006.943353:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777006.943355:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775777006.943358:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.943360:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777006.943363:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777006.943366:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777006.943369:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777006.943371:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777006.943373:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777006.943376:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943389:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.943392:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943398:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.943399:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777006.943402:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777006.943405:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777006.943408:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777006.943409:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313394705 00000100:00000001:0.0:1775777006.943411:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313395959 00000100:00000001:0.0:1775777006.943413:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313394705 00000100:00000001:0.0:1775777006.943415:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313395959 00000100:00000001:0.0:1775777006.943417:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777006.943419:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.943421:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777006.943424:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777006/real 1775777006] req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777006.943430:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777006.943431:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777006.943433:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777006.943435:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777006.943436:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777006.943438:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777006.943441:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777006.943448:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943454:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777006.943456:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.943458:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.943459:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777006.943462:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943469:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777006.943472:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943479:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777006.943481:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.943483:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.943484:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.943486:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777006.943488:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777006.943490:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777006.943493:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777006.943495:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777006.943498:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777006.943506:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822336:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777006.943510:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777006.943514:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943520:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777006.943522:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.943524:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777006.943526:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777006.943528:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777006.943531:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822464:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777006.943534:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777006.943536:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777006.943537:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777006.943539:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777006.943542:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775777006.943544:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777006.943546:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777006.943548:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777006.943551:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777006.943554:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099822464, portal 4 00000100:00000001:0.0:1775777006.943555:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777006.943557:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751265152 : -114841958286464 : ffff978d45defb80) 00000100:00000040:0.0:1775777006.943562:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943568:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777006.943571:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777006.943572:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777006.943575:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099822464, offset 0 00000400:00000200:0.0:1775777006.943579:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777006.943582:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.943585:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777006.943587:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777006.943590:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777006.943592:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777006.943594:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777006.943597:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4656:4656:256:4294967295] 192.168.201.139@tcp LPNI seq info [4656:4656:8:4294967295] 00000400:00000200:0.0:1775777006.943613:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777006.943620:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777006.943624:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.943627:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777006.943632:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777006.943634:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775777006.943638:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777006.943641:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775777006.943644:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777006.943646:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777006.943649:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777006.943652:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777006.943655:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777006.943657:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777006.943659:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777006.943662:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943670:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.943673:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.943679:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.943681:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777006.943683:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777006.943686:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777006.943689:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777006.943691:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313394696 00000100:00000001:0.0:1775777006.943693:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313395959 00000100:00000001:0.0:1775777006.943695:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313394696 00000100:00000001:0.0:1775777006.943697:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313395959 00000100:00000001:0.0:1775777006.943699:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777006.943701:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.943703:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777006.943706:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777006/real 1775777006] req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777006.943713:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777006.945331:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777006.945337:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777006.945344:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777006.945346:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777006.945351:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777006.945359:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777006.945369:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.945376:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777006.945378:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.945381:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.945382:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777006.945385:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.945392:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777006.945397:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.945403:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777006.945406:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.945408:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777006.945409:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.945411:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777006.945413:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777006.945416:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777006.945431:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777006.945434:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777006.945437:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777006.945449:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822464:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777006.945453:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.945456:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.945459:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedc00 x1862033099821952/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.945465:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777006.945468:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777006.945471:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777006.945474:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777006.945476:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777006.945478:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777006.945480:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777006.945483:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158c00. 02000000:00000001:0.0:1775777006.945486:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777006.945488:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777006.945490:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedc00. 00000100:00000001:0.0:1775777006.945494:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777006.945495:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.945497:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.945500:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec700 x1862033099822080/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.945507:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777006.945508:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777006.945510:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777006.945511:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777006.945513:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777006.945514:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777006.945515:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777006.945517:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:0.0:1775777006.945519:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777006.945520:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777006.945521:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec700. 00000100:00000001:0.0:1775777006.945523:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777006.945524:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.945526:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.945530:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deed80 x1862033099822208/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.945536:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777006.945537:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777006.945538:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777006.945541:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777006.945542:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777006.945544:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777006.945545:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777006.945547:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:0.0:1775777006.945549:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777006.945550:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777006.945551:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deed80. 00000100:00000001:0.0:1775777006.945554:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777006.945555:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.945557:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.945560:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def100 x1862033099822336/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.945565:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777006.945566:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777006.945567:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775777006.945569:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777006.945570:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777006.945571:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777006.945572:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777006.945573:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:0.0:1775777006.945575:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777006.945576:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777006.945577:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def100. 00000100:00000001:0.0:1775777006.945579:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777006.945580:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.945582:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777006.945584:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45defb80 x1862033099822464/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777006.945590:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777006.945591:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777006.945593:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775777006.945595:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777006.945596:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777006.945597:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777006.945598:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777006.945600:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:0.0:1775777006.945601:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777006.945994:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777006.945997:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45defb80. 00000100:00000001:0.0:1775777006.946000:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777006.946001:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.946004:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777006.946011:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777006.946013:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777006.946015:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777006.946017:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777007.131952:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777007.131958:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777007.131960:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777007.132001:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777007.132006:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777007.132009:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777007.132011:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777007.132014:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777007.132017:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777007.643932:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777007.643942:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777007.643945:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777007.643952:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777007.643985:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777007.643991:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777007.643995:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777007.643998:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777007.644001:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777007.708359:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777007.708366:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777007.708373:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777007.708375:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777007.708379:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777007.708381:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777007.708385:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777007.708386:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777007.708740:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777007.708747:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777007.708752:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777008.157647:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777008.157653:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777008.157656:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777008.157661:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777008.157664:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777008.157678:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777008.157681:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777008.157683:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777008.157686:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777008.219910:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313395959, at 4313395968 00000100:00000010:1.0:1775777008.219925:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237120. 00000100:00000040:0.0:1775777008.220036:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313395959, at 4313395968 00000100:00000010:0.0:1775777008.220043:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775777008.668523:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777008.668533:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777008.668535:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777008.668543:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777008.668547:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777008.668552:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777008.668555:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777008.668558:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777008.668561:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777008.732505:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777008.732512:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777008.732514:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777008.732517:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777008.732530:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777008.732533:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777008.732535:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777008.732537:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777008.732542:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777008.732543:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777008.732546:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777008.732548:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777008.732551:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777008.732552:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777008.732747:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777008.732753:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777008.732758:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777009.180000:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777009.180009:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777009.180011:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777009.180030:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777009.180035:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777009.180038:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777009.180041:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777009.180044:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777009.180047:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777009.691908:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777009.691915:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777009.691917:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777009.691935:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777009.691947:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777009.691951:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777009.691954:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777009.691956:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777009.691961:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777009.755885:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777009.755890:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777009.755897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777009.755898:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777009.755902:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777009.755904:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777009.755907:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777009.755908:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777009.756077:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777009.756081:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777009.756085:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777010.203968:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777010.203974:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777010.203976:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777010.203981:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777010.203984:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777010.203987:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777010.203990:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777010.203992:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777010.203996:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777010.715915:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777010.715923:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777010.715926:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777010.715931:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777010.715934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777010.715947:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777010.715949:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777010.715951:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777010.715953:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777010.781265:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777010.781271:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777010.781276:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777010.781277:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777010.781280:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777010.781281:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777010.781285:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777010.781285:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777010.781536:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777010.781539:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777010.781542:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777011.230459:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777011.230466:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777011.230468:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777011.230473:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777011.230476:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777011.230479:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777011.230482:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777011.230484:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777011.230488:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777011.739926:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777011.739935:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777011.739937:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777011.739943:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777011.739947:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777011.739951:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777011.739954:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777011.739956:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777011.739959:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804032:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777011.804038:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804055:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777011.804058:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777011.804061:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777011.804064:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804067:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777011.804069:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804071:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804073:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804077:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777011.804079:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777011.804080:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777011.804081:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804083:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777011.804084:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804085:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804087:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804090:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777011.804091:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777011.804092:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777011.804093:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804094:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777011.804095:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804096:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777011.804098:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777011.806533:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777011.806542:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777011.806548:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775777012.059979:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777012.059990:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775777012.059993:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777012.059997:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777012.060002:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777012.060009:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18727 00000400:00000200:2.0:1775777012.060020:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777012.060026:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777012.060029:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777012.060032:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060036:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060039:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060041:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777012.060045:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777012.060047:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775777012.060049:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777012.060052:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775777012.060054:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775777012.060056:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775777012.060059:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060060:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060062:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060063:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777012.060066:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775777012.060069:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060072:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777012.060075:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777012.060079:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777012.060082:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060085:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775777012.060087:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777012.060097:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f480. 00000020:00000040:2.0:1775777012.060101:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775777012.060105:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777012.060107:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060110:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777012.060112:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777012.060116:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777012.060124:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775777012.060129:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777012.060138:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777012.060148:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777012.060153:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777012.060158:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002c91b8b4] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777012.060180:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777012.060183:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777012.060186:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775777012.060188:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777012.060189:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777012.060191:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777012.060193:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18727 00000400:00000200:2.0:1775777012.060196:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777012.060199:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777012.060201:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777012.060203:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060205:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060207:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060209:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777012.060211:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777012.060212:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775777012.060213:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777012.060215:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775777012.060217:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775777012.060218:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775777012.060221:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060222:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060224:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060225:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777012.060227:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775777012.060229:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060233:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777012.060234:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777012.060239:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777012.060241:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060243:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777012.060246:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777012.060249:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060250:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777012.060254:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c380. 00000020:00000040:2.0:1775777012.060256:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775777012.060258:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777012.060259:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060261:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777012.060262:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777012.060263:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777012.060266:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506400. 00000100:00000001:2.0:1775777012.060270:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777012.060275:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777012.060281:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777012.060283:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777012.060287:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f231afe] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777012.060294:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777012.060297:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777012.060300:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775777012.060302:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777012.060303:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777012.060306:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777012.060308:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18727 00000400:00000200:2.0:1775777012.060311:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777012.060314:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777012.060316:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777012.060319:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060321:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060322:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060324:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777012.060325:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777012.060326:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775777012.060328:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777012.060329:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775777012.060331:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775777012.060333:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775777012.060336:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060338:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060339:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060341:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777012.060342:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775777012.060345:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060347:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777012.060348:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777012.060351:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777012.060353:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060355:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777012.060357:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777012.060359:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060361:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777012.060364:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7df80. 00000020:00000040:2.0:1775777012.060366:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775777012.060368:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777012.060369:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060371:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777012.060373:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777012.060374:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777012.060378:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505c00. 00000100:00000001:2.0:1775777012.060380:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777012.060385:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777012.060391:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777012.060393:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777012.060397:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009deec8f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777012.060415:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777012.060418:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777012.060421:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775777012.060422:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777012.060424:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777012.060426:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777012.060429:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18727 00000400:00000200:2.0:1775777012.060432:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777012.060435:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777012.060437:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777012.060440:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060442:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060444:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060446:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777012.060448:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777012.060449:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775777012.060451:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777012.060453:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775777012.060455:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775777012.060456:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775777012.060459:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060461:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060462:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060464:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777012.060466:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775777012.060468:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060470:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777012.060472:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777012.060475:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777012.060477:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060479:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777012.060481:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777012.060483:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060485:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777012.060488:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7dc00. 00000020:00000040:2.0:1775777012.060491:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775777012.060493:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777012.060494:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060495:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777012.060497:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777012.060498:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777012.060514:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506c00. 00000100:00000001:2.0:1775777012.060517:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777012.060522:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777012.060530:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777012.060532:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777012.060536:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f1188f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777012.060542:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777012.060545:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777012.060548:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775777012.060550:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777012.060551:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777012.060553:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777012.060556:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18727 00000400:00000200:2.0:1775777012.060559:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777012.060562:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777012.060564:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777012.060566:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060568:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060570:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060572:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777012.060573:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777012.060574:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775777012.060576:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777012.060577:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775777012.060579:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775777012.060581:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775777012.060584:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060586:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777012.060587:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060589:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777012.060591:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775777012.060594:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777012.060596:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777012.060597:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777012.060600:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777012.060602:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060604:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777012.060605:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777012.060608:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060610:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777012.060613:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d880. 00000020:00000040:2.0:1775777012.060615:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775777012.060617:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777012.060619:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777012.060620:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777012.060621:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777012.060623:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777012.060626:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:2.0:1775777012.060629:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777012.060634:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777012.060641:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777012.060643:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777012.060646:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f88f35f] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777012.060653:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777012.060657:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18737) 00000100:00000001:0.0:1775777012.062598:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777012.062606:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777012.062610:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777012.062621:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.062633:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777012.062635:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.062639:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777012.062642:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777012.062645:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777012.062648:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099822592:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777012.062653:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777012.062658:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777012.062661:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.062663:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777012.062672:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777012.062674:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777012.062682:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777012.062687:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777012.062693:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777012.062702:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099822592, portal 25 00000100:00000001:0.0:1775777012.062704:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777012.062706:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724096 : -114841420827520 : ffff978d65e7f480) 00000100:00000040:0.0:1775777012.062711:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.062718:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777012.062721:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777012.062724:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777012.062726:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099822592, offset 0 00000400:00000200:0.0:1775777012.062733:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777012.062742:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.062749:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777012.062754:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777012.062758:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777012.062762:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777012.062764:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777012.062768:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4657:4657:256:4294967295] 192.168.201.139@tcp LPNI seq info [4657:4657:8:4294967295] 00000400:00000200:0.0:1775777012.062772:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.062812:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777012.062819:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.062824:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777012.062832:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777012.062835:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777012.062841:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777012.062843:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777012.062847:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.062850:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777012.062856:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777012.062860:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777012.062865:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777012.062867:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777012.062872:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777012.062876:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.062897:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.062901:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.062907:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.062909:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777012.062915:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777012.062919:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777012.062921:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313401080 00000100:00000010:0.0:1775777012.062927:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777012.062943:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777012.062950:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313401080 00000100:00000010:0.0:1775777012.062953:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237660. 00000100:00000001:0.0:1775777012.062956:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777012.062959:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777012.062961:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.062963:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777012.062967:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777012/real 1775777012] req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777012.062976:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777012.062978:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777012.062980:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777012.062982:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777012.062998:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777012.063001:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777012.063005:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777012.063013:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063020:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777012.063022:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063024:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.063026:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777012.063029:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063036:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777012.063040:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063047:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777012.063050:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063052:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.063053:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063055:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777012.063057:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777012.063061:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777012.063065:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777012.063067:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777012.063071:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777012.063078:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099822592:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777012.063083:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777012.063086:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063098:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777012.063100:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.063102:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777012.063104:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777012.063105:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777012.063108:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822720:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777012.063111:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777012.063113:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777012.063115:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.063116:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777012.063120:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777012.063122:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777012.063125:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777012.063127:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777012.063131:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777012.063133:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099822720, portal 10 00000100:00000001:0.0:1775777012.063136:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777012.063137:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288711552 : -114841420840064 : ffff978d65e7c380) 00000100:00000040:0.0:1775777012.063143:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063149:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777012.063151:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777012.063153:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777012.063156:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099822720, offset 0 00000400:00000200:0.0:1775777012.063160:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777012.063164:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.063167:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777012.063170:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777012.063172:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777012.063174:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777012.063175:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777012.063178:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4658:4658:256:4294967295] 192.168.201.139@tcp LPNI seq info [4658:4658:8:4294967295] 00000400:00000200:0.0:1775777012.063181:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.063187:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777012.063191:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.063194:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777012.063197:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777012.063199:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777012.063215:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777012.063218:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777012.063222:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.063224:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777012.063227:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777012.063230:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777012.063232:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777012.063234:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777012.063236:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777012.063239:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063246:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.063248:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063254:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.063256:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777012.063259:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777012.063263:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777012.063266:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777012.063267:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313399815 00000100:00000001:0.0:1775777012.063270:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313401080 00000100:00000001:0.0:1775777012.063272:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313399815 00000100:00000001:0.0:1775777012.063274:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313401080 00000100:00000001:0.0:1775777012.063276:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777012.063278:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.063280:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777012.063283:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777012/real 1775777012] req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777012.063291:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777012.063292:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777012.063294:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777012.063296:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777012.063297:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777012.063299:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777012.063302:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777012.063309:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063315:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777012.063317:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063318:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.063320:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777012.063323:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063329:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777012.063332:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063338:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777012.063340:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063342:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.063343:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063344:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777012.063346:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777012.063348:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777012.063352:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777012.063353:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777012.063357:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777012.063364:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822720:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777012.063367:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777012.063371:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063376:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777012.063378:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.063379:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777012.063381:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777012.063383:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777012.063385:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822848:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777012.063388:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777012.063390:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777012.063391:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.063393:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777012.063396:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777012.063398:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777012.063401:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777012.063403:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777012.063405:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777012.063408:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099822848, portal 10 00000100:00000001:0.0:1775777012.063409:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777012.063411:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288718720 : -114841420832896 : ffff978d65e7df80) 00000100:00000040:0.0:1775777012.063416:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063421:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777012.063423:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777012.063425:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777012.063428:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099822848, offset 0 00000400:00000200:0.0:1775777012.063431:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777012.063435:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.063438:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777012.063440:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777012.063442:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777012.063444:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777012.063446:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777012.063449:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4659:4659:256:4294967295] 192.168.201.139@tcp LPNI seq info [4659:4659:8:4294967295] 00000400:00000200:0.0:1775777012.063452:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.063457:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777012.063461:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.063464:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777012.063467:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777012.063470:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777012.063473:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777012.063476:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777012.063479:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.063481:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777012.063483:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777012.063486:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777012.063489:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777012.063491:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777012.063493:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777012.063497:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063504:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.063506:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063512:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.063514:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777012.063517:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777012.063519:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777012.063522:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777012.063524:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313399816 00000100:00000001:0.0:1775777012.063526:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313401080 00000100:00000001:0.0:1775777012.063528:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313399816 00000100:00000001:0.0:1775777012.063530:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313401080 00000100:00000001:0.0:1775777012.063532:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777012.063534:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.063535:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777012.063539:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777012/real 1775777012] req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777012.063546:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777012.063547:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777012.063548:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777012.063551:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777012.063552:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777012.063554:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777012.063557:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777012.063564:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063570:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777012.063571:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063573:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.063574:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777012.063577:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063583:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777012.063586:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063592:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777012.063594:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063595:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.063597:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063598:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777012.063599:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777012.063602:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777012.063605:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777012.063607:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777012.063610:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777012.063617:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822848:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777012.063620:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777012.063623:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063630:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777012.063631:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.063633:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777012.063634:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777012.063636:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777012.063639:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822976:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777012.063642:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777012.063644:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777012.063645:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.063647:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777012.063650:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777012.063651:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777012.063654:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777012.063656:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777012.063659:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777012.063661:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099822976, portal 4 00000100:00000001:0.0:1775777012.063662:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777012.063663:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288717824 : -114841420833792 : ffff978d65e7dc00) 00000100:00000040:0.0:1775777012.063668:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063673:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777012.063675:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777012.063677:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777012.063679:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099822976, offset 0 00000400:00000200:0.0:1775777012.063682:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777012.063685:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.063688:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777012.063690:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777012.063692:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777012.063694:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777012.063696:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777012.063698:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4660:4660:256:4294967295] 192.168.201.139@tcp LPNI seq info [4660:4660:8:4294967295] 00000400:00000200:0.0:1775777012.063701:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.063706:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777012.063709:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.063712:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777012.063715:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777012.063718:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777012.063720:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777012.063722:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777012.063726:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.063727:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777012.063730:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777012.063733:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777012.063735:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777012.063737:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777012.063739:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777012.063742:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063748:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.063751:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063757:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.063758:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777012.063761:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777012.063764:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777012.063767:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777012.063768:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313399825 00000100:00000001:0.0:1775777012.063771:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313401080 00000100:00000001:0.0:1775777012.063773:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313399825 00000100:00000001:0.0:1775777012.063775:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313401080 00000100:00000001:0.0:1775777012.063777:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777012.063803:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.063805:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777012.063809:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777012/real 1775777012] req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777012.063816:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777012.063817:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777012.063819:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777012.063821:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777012.063822:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777012.063824:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777012.063827:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777012.063834:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063840:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777012.063841:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063842:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.063844:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777012.063846:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063852:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777012.063855:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063861:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777012.063863:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063864:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.063865:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.063867:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777012.063868:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777012.063870:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777012.063873:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777012.063874:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777012.063877:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777012.063899:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099822976:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777012.063903:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777012.063906:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063912:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777012.063914:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.063916:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777012.063918:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777012.063920:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777012.063922:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823104:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777012.063925:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777012.063927:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777012.063928:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777012.063930:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777012.063933:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777012.063934:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777012.063937:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777012.063940:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777012.063942:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777012.063945:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099823104, portal 4 00000100:00000001:0.0:1775777012.063947:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777012.063948:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716928 : -114841420834688 : ffff978d65e7d880) 00000100:00000040:0.0:1775777012.063953:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.063958:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777012.063960:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777012.063962:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777012.063964:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099823104, offset 0 00000400:00000200:0.0:1775777012.063967:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777012.063971:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.063974:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777012.063977:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777012.063978:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777012.063980:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777012.063982:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777012.063984:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4661:4661:256:4294967295] 192.168.201.139@tcp LPNI seq info [4661:4661:8:4294967295] 00000400:00000200:0.0:1775777012.063987:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777012.063992:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777012.063997:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.064000:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777012.064004:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777012.064006:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777012.064010:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777012.064013:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777012.064016:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777012.064018:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777012.064020:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777012.064023:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777012.064026:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777012.064027:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777012.064029:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777012.064032:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064039:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.064042:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064047:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.064049:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777012.064051:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777012.064053:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777012.064056:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777012.064058:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313399817 00000100:00000001:0.0:1775777012.064060:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313401080 00000100:00000001:0.0:1775777012.064062:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313399817 00000100:00000001:0.0:1775777012.064064:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313401080 00000100:00000001:0.0:1775777012.064066:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777012.064067:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.064069:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777012.064072:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777012/real 1775777012] req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777012.064079:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777012.064080:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777012.064082:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777012.064083:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777012.064085:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777012.064087:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777012.064089:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777012.064097:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064102:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777012.064103:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064105:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.064106:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777012.064109:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064114:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777012.064117:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064123:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777012.064125:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064126:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777012.064127:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064129:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777012.064130:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777012.064133:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777012.064136:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777012.064138:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777012.064141:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777012.064148:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823104:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777012.064151:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064153:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.064156:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f480 x1862033099822592/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064162:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777012.064165:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777012.064167:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777012.064170:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777012.064172:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777012.064174:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777012.064176:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777012.064178:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:0.0:1775777012.064181:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777012.064183:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777012.064185:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f480. 00000100:00000001:0.0:1775777012.064188:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777012.064189:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064190:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.064193:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099822720/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064198:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777012.064199:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777012.064201:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777012.064203:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777012.064203:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777012.064204:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777012.064206:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777012.064207:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506400. 02000000:00000001:0.0:1775777012.064208:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777012.064209:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777012.064211:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:0.0:1775777012.064213:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777012.064214:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064215:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.064218:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7df80 x1862033099822848/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064223:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777012.064225:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777012.064226:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777012.064227:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777012.064228:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777012.064229:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777012.064231:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777012.064232:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505c00. 02000000:00000001:0.0:1775777012.064233:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777012.064234:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777012.064235:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7df80. 00000100:00000001:0.0:1775777012.064237:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777012.064238:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064239:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.064242:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7dc00 x1862033099822976/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064247:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777012.064247:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777012.064249:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777012.064251:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777012.064252:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777012.064253:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777012.064255:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777012.064256:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:0.0:1775777012.064258:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777012.064259:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777012.064260:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7dc00. 00000100:00000001:0.0:1775777012.064262:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777012.064263:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064264:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777012.064267:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d880 x1862033099823104/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777012.064272:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777012.064273:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777012.064274:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777012.064276:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777012.064277:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777012.064277:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777012.064279:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777012.064280:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:0.0:1775777012.064282:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777012.064283:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777012.064284:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d880. 00000100:00000001:0.0:1775777012.064286:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777012.064287:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064289:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777012.064295:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777012.064297:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777012.064299:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777012.064300:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777012.251995:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777012.252002:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777012.252004:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777012.252010:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777012.252013:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777012.252016:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777012.252019:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777012.252021:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777012.252024:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777012.766927:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777012.766940:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777012.766942:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777012.766948:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777012.766952:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777012.766957:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777012.766960:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777012.766969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777012.766982:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777012.828546:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777012.828552:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777012.828557:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777012.828558:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777012.828561:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777012.828562:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777012.828565:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777012.828566:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777012.828715:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777012.828719:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777012.828722:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777013.278484:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777013.278491:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777013.278493:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777013.278498:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777013.278502:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777013.278506:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777013.278509:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777013.278512:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777013.278515:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777013.341380:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313401080, at 4313401089 00000100:00000010:0.0:1775777013.341396:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775777013.342472:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313401080, at 4313401090 00000100:00000010:1.0:1775777013.342482:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237660. 00000100:00000001:3.0:1775777013.795450:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777013.795458:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777013.795460:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777013.795465:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777013.795468:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777013.795472:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777013.795475:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777013.795477:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777013.795480:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777013.852252:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777013.852261:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777013.852270:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777013.852271:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777013.852275:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777013.852276:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777013.852281:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777013.852283:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777013.853518:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777013.853523:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777013.853528:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777014.304570:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777014.304578:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777014.304580:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777014.304586:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777014.304590:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777014.304594:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777014.304597:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777014.304599:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777014.304602:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777014.811952:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777014.811959:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777014.811960:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777014.811966:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777014.811970:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777014.811974:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777014.811976:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777014.811979:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777014.811982:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777014.876040:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777014.876047:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777014.876053:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777014.876054:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777014.876058:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777014.876059:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777014.876062:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777014.876063:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777014.876577:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777014.876581:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777014.876584:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777015.325327:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777015.325334:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777015.325336:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777015.325342:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777015.325346:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777015.325349:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777015.325352:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777015.325354:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777015.325357:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777015.836986:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777015.836992:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777015.836993:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777015.837012:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777015.837016:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777015.837020:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777015.837022:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777015.837024:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777015.837027:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777015.899874:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777015.899880:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777015.899886:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777015.899887:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777015.899891:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777015.899892:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777015.899897:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777015.899898:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777015.899989:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777015.899993:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777015.899996:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777016.349473:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777016.349479:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777016.349481:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777016.349487:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777016.349490:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777016.349498:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777016.349501:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777016.349504:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777016.349506:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777016.859941:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777016.859951:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777016.859953:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777016.859960:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777016.859964:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777016.859969:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777016.859971:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777016.859974:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777016.859976:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777016.924594:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777016.924599:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777016.924605:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777016.924617:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777016.924621:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777016.924622:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777016.924626:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777016.924627:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777016.924749:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777016.924753:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777016.924756:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775777017.187760:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777017.187772:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775777017.187775:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777017.187820:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777017.187826:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777017.187835:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18732 00000400:00000200:3.0:1775777017.187845:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777017.187852:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777017.187855:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777017.187858:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.187862:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.187865:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.187867:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777017.187871:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777017.187873:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775777017.187876:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777017.187879:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775777017.187881:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775777017.187883:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775777017.187886:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.187888:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.187890:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.187892:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777017.187894:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775777017.188013:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777017.188016:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777017.188019:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777017.188022:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777017.188025:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.188027:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775777017.188029:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777017.188038:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775777017.188042:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775777017.188046:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777017.188048:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.188051:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777017.188053:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777017.188057:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777017.188065:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775777017.188070:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777017.188079:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777017.189033:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777017.189038:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777017.189046:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777017.189075:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.189334:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777017.189339:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777017.189342:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777017.189348:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.189355:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777017.189358:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.189362:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777017.189364:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777017.189367:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777017.189370:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099823232:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777017.189373:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777017.189378:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777017.189380:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.189383:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777017.189391:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777017.189393:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777017.189400:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777017.189404:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777017.189411:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777017.189418:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099823232, portal 25 00000100:00000001:0.0:1775777017.189420:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777017.189422:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751253504 : -114841958298112 : ffff978d45dece00) 00000100:00000040:0.0:1775777017.189427:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.189434:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777017.189436:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777017.189439:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777017.189441:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099823232, offset 0 00000400:00000200:0.0:1775777017.189446:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777017.189455:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.189462:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777017.189467:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777017.189471:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777017.189475:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777017.189477:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777017.189481:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4662:4662:256:4294967295] 192.168.201.139@tcp LPNI seq info [4662:4662:8:4294967295] 00000400:00000200:0.0:1775777017.189484:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.189494:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777017.189500:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.189505:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777017.189511:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777017.189514:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777017.189519:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777017.189522:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777017.189527:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.189530:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777017.189536:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777017.189540:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777017.189545:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777017.189547:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777017.189552:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777017.189556:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.189565:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.189568:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.189575:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.189576:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777017.189582:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777017.189585:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777017.189588:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313406206 00000100:00000010:0.0:1775777017.189593:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777017.189600:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777017.189618:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313406206 00000100:00000010:0.0:1775777017.189622:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237540. 00000100:00000001:0.0:1775777017.189624:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777017.189628:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777017.189630:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.189633:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777017.189636:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777017/real 1775777017] req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777017.189644:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777017.189646:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777017.189647:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000100:00080000:3.0:1775777017.189998:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777017.190004:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775777017.190006:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777017.190008:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777017.190011:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777017.190015:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18732 00000400:00000200:3.0:1775777017.190020:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777017.190023:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777017.190026:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777017.190029:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190031:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.190033:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190035:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777017.190039:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777017.190040:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775777017.190042:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777017.190045:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775777017.190047:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775777017.190049:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775777017.190052:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190054:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.190055:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190057:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777017.190060:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775777017.190063:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190066:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777017.190068:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777017.190073:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777017.190075:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190077:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775777017.190179:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777017.190181:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000008:00100000:3.0:1775777017.190330:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777017.190333:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190335:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777017.190341:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775777017.190345:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775777017.190348:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777017.190350:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190352:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777017.190354:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777017.190357:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777017.190363:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775777017.190368:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777017.190373:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777017.190380:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777017.190383:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777017.190387:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777017.190396:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777017.190398:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777017.190401:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775777017.190403:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777017.190404:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777017.190406:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777017.190409:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18732 00000400:00000200:3.0:1775777017.190413:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777017.190416:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777017.190418:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777017.190420:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190422:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.190424:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190426:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777017.190428:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777017.190429:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775777017.190431:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777017.190433:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775777017.190435:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775777017.190437:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775777017.190440:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190441:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.190443:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190445:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777017.190448:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775777017.190450:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190453:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777017.190455:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777017.190458:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777017.190460:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190462:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777017.190464:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777017.190466:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190468:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777017.190471:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775777017.190473:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775777017.190476:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777017.190477:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190479:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777017.190480:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777017.190482:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777017.190485:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775777017.190488:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777017.190492:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777017.190500:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777017.190502:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777017.190505:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777017.190513:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777017.190515:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777017.190518:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775777017.190520:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777017.190521:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777017.190524:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777017.190526:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18732 00000400:00000200:3.0:1775777017.190530:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777017.190533:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777017.190535:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777017.190537:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190539:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.190541:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190543:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777017.190545:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777017.190546:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775777017.190548:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777017.190549:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775777017.190552:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775777017.190554:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775777017.190557:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190559:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.190561:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190563:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777017.190565:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775777017.190567:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777017.190570:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777017.190572:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777017.190574:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777017.190577:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190579:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777017.190580:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777017.190583:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190584:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777017.190588:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775777017.190590:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775777017.190593:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777017.190594:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.190596:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777017.190597:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777017.190599:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777017.190603:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775777017.190606:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777017.190610:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777017.190617:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777017.190619:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777017.190623:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777017.190630:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777017.190633:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777017.190636:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775777017.190637:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777017.190639:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777017.190641:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777017.190643:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18732 00000400:00000200:3.0:1775777017.190647:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777017.190985:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777017.190991:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777017.191000:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.191006:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777017.191008:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.191011:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.191012:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777017.191015:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.191021:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777017.191026:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.191032:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777017.191034:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.191036:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.191037:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.191040:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777017.191041:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777017.191044:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777017.191047:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777017.191049:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777017.191053:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777017.191059:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099823232:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777017.191063:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.191066:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.191068:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dece00 x1862033099823232/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.191079:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777017.192280:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777017.192284:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777017.192291:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777017.192293:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777017.192295:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777017.192297:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777017.192299:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:0.0:1775777017.192303:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777017.192305:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777017.192306:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dece00. 00000100:00000001:0.0:1775777017.192310:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777017.192311:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.192314:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.192321:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777017.192323:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.192325:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777017.192327:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777017.192330:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777017.192338:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.192345:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777017.192347:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.192350:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777017.192353:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777017.192355:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777017.192358:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823360:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777017.192362:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777017.192365:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777017.192368:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.192370:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777017.192375:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777017.192377:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777017.192383:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777017.192387:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777017.192392:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777017.192397:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099823360, portal 10 00000100:00000001:0.0:1775777017.192399:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777017.192400:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751259776 : -114841958291840 : ffff978d45dee680) 00000100:00000040:0.0:1775777017.192406:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.192412:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777017.192414:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777017.192416:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777017.192419:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099823360, offset 0 00000400:00000200:0.0:1775777017.192425:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777017.192431:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.192438:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777017.192443:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777017.192447:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777017.192450:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777017.192452:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777017.192456:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4663:4663:256:4294967295] 192.168.201.139@tcp LPNI seq info [4663:4663:8:4294967295] 00000400:00000200:0.0:1775777017.192460:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.192468:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777017.192473:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.192477:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777017.192482:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777017.192485:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777017.192490:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777017.192493:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777017.192496:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.192498:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777017.192502:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777017.192506:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777017.192510:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777017.192512:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777017.192515:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777017.192518:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.192525:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.192528:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.192533:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.192535:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777017.192539:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777017.192542:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777017.192545:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777017.192547:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313404942 00000100:00000001:0.0:1775777017.192550:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313406206 00000100:00000001:0.0:1775777017.192551:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313404942 00000100:00000001:0.0:1775777017.192553:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313406206 00000100:00000001:0.0:1775777017.192555:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777017.192557:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.192559:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777017.192562:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777017/real 1775777017] req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777017.192569:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777017.192572:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777017.192573:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777017.192575:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777017.192577:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777017.192579:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777017.192581:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777017.192589:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.192598:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777017.192600:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.192602:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.192603:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777017.192606:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.192612:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777017.192615:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.192622:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777017.192625:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.192626:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.192627:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.192629:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777017.192631:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777017.192634:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777017.192637:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777017.192639:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777017.192643:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775777017.193439:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777017.193446:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777017.193452:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.193456:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.193457:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.193460:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777017.193463:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777017.193465:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775777017.193467:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777017.193471:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775777017.193473:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775777017.193475:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775777017.193477:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.193479:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777017.193481:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777017.193483:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777017.193485:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775777017.193488:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777017.193490:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777017.193492:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777017.193498:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777017.193500:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.193502:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777017.193506:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777017.193509:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.193511:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777017.193518:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775777017.193521:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775777017.193525:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777017.193528:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.193530:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777017.193533:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777017.193536:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777017.193542:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775777017.193548:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777017.193554:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777017.193563:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777017.193568:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777017.193572:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777017.193583:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777017.193589:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18742) 00000100:00100000:0.0:1775777017.194875:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823360:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777017.194886:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777017.194892:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.195876:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777017.195878:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.195883:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777017.195886:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777017.195889:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777017.195894:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823488:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777017.196164:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777017.196169:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777017.196172:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.196175:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777017.196406:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777017.196411:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777017.196420:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777017.196427:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777017.196435:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777017.196443:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099823488, portal 10 00000100:00000001:0.0:1775777017.196445:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777017.196447:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751252608 : -114841958299008 : ffff978d45deca80) 00000100:00000040:0.0:1775777017.196457:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.196465:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777017.196467:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777017.196470:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777017.196472:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099823488, offset 0 00000400:00000200:0.0:1775777017.196478:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777017.196488:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.196497:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777017.196502:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777017.196506:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777017.196511:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777017.196514:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777017.196518:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4664:4664:256:4294967295] 192.168.201.139@tcp LPNI seq info [4664:4664:8:4294967295] 00000400:00000200:0.0:1775777017.196523:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.196533:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777017.196539:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.196545:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777017.196565:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777017.196569:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777017.196575:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777017.196578:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777017.196583:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.196585:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777017.196592:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777017.196596:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777017.196601:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777017.196603:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777017.196617:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777017.196622:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.196632:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.196635:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.196640:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.196642:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777017.196647:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777017.198020:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777017.198031:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777017.198038:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313404951 00000100:00000001:0.0:1775777017.198041:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313406206 00000100:00000001:0.0:1775777017.198044:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313404951 00000100:00000001:0.0:1775777017.198045:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313406206 00000100:00000001:0.0:1775777017.198047:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777017.198051:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.198054:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777017.198061:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777017/real 1775777017] req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777017.198075:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777017.198076:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777017.198079:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777017.198740:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777017.198743:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777017.198747:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777017.198753:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777017.198763:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.198770:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777017.198772:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.198775:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.198776:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777017.198818:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.198824:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777017.198829:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.198835:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777017.198838:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.198840:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.198841:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.198843:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777017.198845:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777017.198849:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777017.198853:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777017.198856:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777017.198860:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777017.198868:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823488:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777017.198874:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777017.198877:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.198884:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777017.198886:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.198890:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777017.198893:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777017.199772:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777017.199826:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823616:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777017.199832:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777017.199838:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777017.199840:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.199842:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777017.199854:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777017.199856:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777017.199865:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777017.199871:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777017.199878:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777017.199887:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099823616, portal 4 00000100:00000001:0.0:1775777017.199890:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777017.199891:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751250816 : -114841958300800 : ffff978d45dec380) 00000100:00000040:0.0:1775777017.200401:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.200411:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777017.200414:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777017.200419:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777017.200425:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099823616, offset 0 00000400:00000200:0.0:1775777017.200434:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777017.200446:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.200456:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777017.200463:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777017.200468:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777017.200473:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777017.200476:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777017.200482:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4665:4665:256:4294967295] 192.168.201.139@tcp LPNI seq info [4665:4665:8:4294967295] 00000400:00000200:0.0:1775777017.200487:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.200499:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777017.200505:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.200510:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777017.200520:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777017.200524:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777017.200530:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777017.200533:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777017.200558:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.200561:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777017.200569:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777017.200575:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777017.200580:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777017.200583:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777017.200587:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777017.200590:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.200601:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.200604:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.200623:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.200625:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777017.200632:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777017.200638:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777017.200642:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777017.200646:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313404962 00000100:00000001:0.0:1775777017.202041:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313406206 00000100:00000001:0.0:1775777017.202053:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313404964 00000100:00000001:0.0:1775777017.202057:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313406206 00000100:00000001:0.0:1775777017.202060:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777017.202064:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.202067:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777017.203170:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777017/real 1775777017] req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777017.203808:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777017.203811:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777017.203814:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777017.203821:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777017.203823:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777017.203828:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777017.203837:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777017.203846:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.203853:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777017.203855:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.203858:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.203860:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777017.203864:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.203870:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777017.203875:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.203882:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777017.203886:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.203888:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.203889:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.203892:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777017.203894:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777017.204311:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777017.204318:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777017.204321:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777017.204325:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777017.204333:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823616:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777017.204339:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.204342:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.204346:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee680 x1862033099823360/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.204352:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777017.204355:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777017.204360:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777017.204365:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777017.204367:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777017.204369:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777017.204372:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777017.204374:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:0.0:1775777017.204378:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777017.204380:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777017.204382:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee680. 00000100:00000001:0.0:1775777017.204385:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777017.204387:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.204389:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.204392:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deca80 x1862033099823488/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.204404:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777017.204406:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777017.204407:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777017.204409:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777017.204410:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777017.204411:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777017.204413:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777017.204414:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:0.0:1775777017.204415:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777017.204416:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777017.204418:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deca80. 00000100:00000001:0.0:1775777017.204420:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777017.204421:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.204422:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.204424:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec380 x1862033099823616/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.204430:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777017.204431:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777017.204433:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777017.204435:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777017.204436:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777017.204437:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777017.204439:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777017.204440:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:0.0:1775777017.204442:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777017.204443:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777017.204444:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec380. 00000100:00000001:0.0:1775777017.204446:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777017.204448:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.204450:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.204458:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777017.204460:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.204463:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777017.204465:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777017.204468:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777017.204472:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.204480:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777017.204482:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.204485:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777017.204488:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777017.204490:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777017.204493:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823744:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777017.204497:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777017.204502:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777017.204504:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777017.204507:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777017.204515:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777017.204517:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777017.204525:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777017.204529:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777017.204534:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777017.204541:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099823744, portal 4 00000100:00000001:0.0:1775777017.204543:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777017.204545:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751249920 : -114841958301696 : ffff978d45dec000) 00000100:00000040:0.0:1775777017.204551:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.204558:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777017.204560:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777017.204563:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777017.204566:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099823744, offset 0 00000400:00000200:0.0:1775777017.204571:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777017.204580:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.204587:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777017.204591:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777017.204596:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777017.204599:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777017.204602:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777017.204617:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4666:4666:256:4294967295] 192.168.201.139@tcp LPNI seq info [4666:4666:8:4294967295] 00000400:00000200:0.0:1775777017.204621:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777017.204630:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777017.204635:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.204640:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777017.204646:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777017.208443:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777017.208465:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777017.208471:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777017.208475:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777017.208478:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777017.208488:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777017.208494:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777017.208503:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777017.208506:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777017.208511:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777017.208519:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.208532:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.208536:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.208543:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.208545:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777017.208554:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777017.208559:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777017.208565:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777017.208569:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313404961 00000100:00000001:0.0:1775777017.208572:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313406206 00000100:00000001:0.0:1775777017.208575:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313404961 00000100:00000001:0.0:1775777017.208577:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313406206 00000100:00000001:0.0:1775777017.208578:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777017.208582:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.208584:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777017.208589:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777017/real 1775777017] req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777017.208599:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777017.208601:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777017.208603:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777017.208617:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777017.208620:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777017.208623:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777017.208627:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777017.208636:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.208642:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777017.208644:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.208646:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.208648:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777017.208848:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.208856:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777017.208861:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.208867:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777017.208870:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.208872:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777017.208873:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.208876:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777017.208878:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777017.208881:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777017.208885:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777017.208888:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777017.208891:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777017.208977:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099823744:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777017.208983:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.208986:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777017.208988:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099823744/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777017.208993:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777017.208996:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777017.209000:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777017.209004:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777017.209006:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777017.209008:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777017.209010:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777017.209012:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115ac00. 02000000:00000001:0.0:1775777017.209017:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777017.209018:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777017.209020:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:0.0:1775777017.209023:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777017.209025:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.209027:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777017.209035:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777017.209037:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777017.209040:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777017.209042:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777017.371966:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777017.371973:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777017.371975:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777017.371982:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777017.371986:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777017.371990:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777017.371992:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777017.371995:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777017.371997:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777017.884091:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777017.884098:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777017.884100:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777017.884106:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777017.884110:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777017.884114:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777017.884116:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777017.884118:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777017.884121:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777017.948292:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777017.948307:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777017.948313:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777017.948314:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777017.948317:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777017.948318:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777017.948322:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777017.948323:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777017.948418:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777017.948423:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777017.948426:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777018.395976:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777018.395982:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777018.395985:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777018.395990:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777018.395994:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777018.395997:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777018.396000:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777018.396002:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777018.396005:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777018.460246:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313406206, at 4313406208 00000100:00000010:0.0:1775777018.460262:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775777018.460366:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313406206, at 4313406208 00000100:00000010:1.0:1775777018.460372:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237540. 00000100:00000001:3.0:1775777018.908331:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777018.908337:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777018.908340:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777018.908345:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777018.908349:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777018.908353:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777018.908356:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777018.908358:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777018.908360:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777018.972862:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777018.972879:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777018.972882:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777018.972886:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777018.972891:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777018.972893:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777018.972895:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777018.972897:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777018.972901:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777018.972902:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777018.972906:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777018.972907:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777018.972910:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777018.972911:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777018.973090:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777018.973094:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777018.973098:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777019.419965:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777019.419971:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777019.419973:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777019.419978:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777019.419982:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777019.419986:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777019.419989:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777019.419991:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777019.419995:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777019.933680:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777019.933687:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777019.933689:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777019.933695:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777019.933698:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777019.933702:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777019.933705:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777019.933707:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777019.933709:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777019.998090:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777019.998096:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777019.998103:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777019.998104:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777019.998108:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777019.998109:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777019.998112:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777019.998113:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777020.000613:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777020.000619:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777020.000625:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777020.449287:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777020.449294:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777020.449296:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777020.449301:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777020.449304:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777020.449308:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777020.449311:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777020.449313:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777020.449316:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777020.959977:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777020.959986:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777020.959988:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777020.959994:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777020.959998:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777020.960003:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777020.960005:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777020.960008:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777020.960011:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777021.021225:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777021.021233:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777021.021240:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777021.021242:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777021.021246:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777021.021247:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777021.021250:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777021.021251:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777021.021369:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777021.021373:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777021.021376:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777021.468099:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777021.468107:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777021.468109:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777021.468115:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777021.468119:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777021.468124:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777021.468127:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777021.468130:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777021.468133:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777021.979928:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777021.979935:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777021.979937:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777021.979943:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777021.979946:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777021.979950:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777021.979953:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777021.979955:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777021.979957:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777022.045936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045942:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777022.045946:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777022.045949:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777022.045953:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045956:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777022.045958:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045960:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045962:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045965:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777022.045967:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777022.045968:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777022.045970:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045971:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777022.045972:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045973:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045975:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045978:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777022.045979:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777022.045980:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777022.045981:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045983:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777022.045984:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045985:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777022.045987:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777022.047765:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777022.047770:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777022.047775:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775777022.300860:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777022.300925:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775777022.300929:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777022.300932:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777022.300936:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777022.300943:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18737 00000400:00000200:2.0:1775777022.300953:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777022.300960:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777022.300962:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777022.300966:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.300969:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.300971:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.300973:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777022.300977:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777022.300979:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775777022.300981:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777022.300984:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775777022.300986:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775777022.300988:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775777022.300990:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.300993:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.300994:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.300996:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777022.300999:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775777022.301001:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777022.301004:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777022.301007:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777022.301010:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777022.301013:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.301015:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775777022.301018:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777022.301028:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f800. 00000020:00000040:2.0:1775777022.301032:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775777022.301035:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777022.301037:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.301040:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777022.301043:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777022.301047:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777022.301056:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506000. 00000100:00000001:2.0:1775777022.301063:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777022.301072:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777022.301083:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777022.301087:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777022.301092:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8818441] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.301165:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777022.301169:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777022.301171:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777022.301176:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.301183:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777022.301185:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.301188:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777022.301191:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777022.301193:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777022.301196:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099823872:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777022.301200:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777022.301204:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777022.301206:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.301208:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777022.301213:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777022.301215:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777022.301230:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777022.301234:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777022.301241:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777022.301247:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099823872, portal 25 00000100:00000001:0.0:1775777022.301250:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777022.301251:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724992 : -114841420826624 : ffff978d65e7f800) 00000100:00000040:0.0:1775777022.301256:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.301263:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777022.301266:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777022.301268:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777022.301271:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099823872, offset 0 00000400:00000200:0.0:1775777022.301277:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777022.301285:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.301290:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777022.301294:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777022.301298:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777022.301302:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777022.301305:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777022.301308:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4667:4667:256:4294967295] 192.168.201.139@tcp LPNI seq info [4667:4667:8:4294967295] 00000400:00000200:0.0:1775777022.301312:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.301321:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777022.301327:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.301332:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777022.301339:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777022.301342:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775777022.301357:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777022.301361:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775777022.301366:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.301368:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777022.301375:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777022.301380:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777022.301385:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777022.301387:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777022.301391:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777022.301395:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777022.301421:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777022.301425:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777022.301428:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775777022.301430:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777022.301431:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777022.301434:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777022.301437:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18737 00000400:00000200:2.0:1775777022.301441:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777022.301444:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777022.301446:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777022.301448:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.301450:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.301452:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.301454:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777022.301457:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777022.301458:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775777022.301459:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777022.301461:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775777022.301463:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775777022.301465:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775777022.301467:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.301469:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.301470:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.301472:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777022.301475:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775777022.301477:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777022.301479:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777022.301481:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777022.301485:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777022.301487:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.301489:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777022.301493:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777022.301495:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.301497:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777022.301501:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ed80. 00000020:00000040:2.0:1775777022.301503:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775777022.301506:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777022.301508:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.301509:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777022.301510:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777022.301512:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777022.301516:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507000. 00000100:00000001:2.0:1775777022.301520:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777022.301524:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777022.301531:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777022.301534:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777022.301537:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006375da9b] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.304377:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.304398:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.304411:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.304413:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777022.304430:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777022.304439:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777022.304442:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313411321 00000100:00000010:0.0:1775777022.304452:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777022.304460:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777022.304469:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313411321 00000100:00000010:0.0:1775777022.304474:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237420. 00000100:00000001:0.0:1775777022.304476:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777022.304480:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777022.304483:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.304486:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777022.304491:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777022/real 1775777022] req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777022.304505:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777022.304507:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777022.304511:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777022.304514:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777022.304517:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777022.304520:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777022.304524:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777022.304533:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.304542:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777022.304544:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.304546:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.304548:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777022.304551:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.304558:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777022.304563:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.304569:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777022.304572:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.304573:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.304574:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.304577:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777022.304579:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777022.304583:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777022.304587:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777022.304590:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777022.304594:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777022.304601:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099823872:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777022.304973:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.304976:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.304981:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f800 x1862033099823872/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.304989:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777022.304992:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777022.304996:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777022.305000:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777022.305003:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777022.305005:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777022.305007:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777022.305009:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506000. 02000000:00000001:0.0:1775777022.305013:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777022.305015:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777022.305017:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f800. 00000100:00000001:0.0:1775777022.305021:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777022.305022:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.305024:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.305033:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777022.305034:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.305037:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777022.305040:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777022.305041:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777022.305046:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.305059:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777022.305061:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.305064:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777022.305067:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777022.305069:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777022.305072:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824000:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777022.305076:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777022.305080:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777022.305082:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.305084:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777022.305090:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777022.305092:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777022.305098:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777022.305102:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777022.305107:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777022.305113:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099824000, portal 10 00000100:00000001:0.0:1775777022.305115:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777022.305116:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288722304 : -114841420829312 : ffff978d65e7ed80) 00000100:00000040:0.0:1775777022.305123:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.305129:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777022.305132:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777022.305134:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777022.305137:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099824000, offset 0 00000400:00000200:0.0:1775777022.305142:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777022.305149:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.305155:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777022.305160:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777022.305165:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777022.305169:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777022.305171:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777022.305176:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4668:4668:256:4294967295] 192.168.201.139@tcp LPNI seq info [4668:4668:8:4294967295] 00000400:00000200:0.0:1775777022.305180:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.305190:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777022.305195:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.305200:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777022.305207:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777022.305210:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775777022.305215:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777022.305287:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775777022.305293:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.305295:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777022.305300:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777022.305305:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777022.305310:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777022.305312:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777022.305315:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777022.305319:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.305327:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.305330:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.305336:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.305338:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777022.305341:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777022.305346:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777022.305349:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777022.305351:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313410057 00000100:00000001:0.0:1775777022.305354:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313411321 00000100:00000001:0.0:1775777022.305356:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313410057 00000100:00000001:0.0:1775777022.305358:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313411321 00000100:00000001:0.0:1775777022.305360:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777022.305362:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.305364:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777022.305367:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777022/real 1775777022] req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777022.305375:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777022.305376:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777022.305378:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777022.305381:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777022.305382:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777022.305384:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777022.305387:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777022.305395:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.305401:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777022.305402:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.305404:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.305406:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777022.305409:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.305415:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777022.305418:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.305425:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777022.305427:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.305429:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.305430:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.305432:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777022.305434:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777022.305437:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777022.305442:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777022.305444:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777022.305447:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777022.305455:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824000:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777022.305459:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.305460:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.305463:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ed80 x1862033099824000/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.305468:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777022.305470:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777022.305472:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777022.305475:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777022.305476:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777022.305478:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777022.305479:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777022.305481:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507000. 02000000:00000001:0.0:1775777022.305483:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777022.305484:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777022.305485:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ed80. 00000100:00000001:0.0:1775777022.305488:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777022.305489:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.305491:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.305495:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777022.305496:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.305498:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777022.305500:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307445:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777022.307455:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777022.307460:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775777022.307462:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777022.307465:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777022.307469:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777022.307476:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18737 00000400:00000200:2.0:1775777022.307485:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777022.307492:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777022.307495:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777022.307498:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307501:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.307503:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307505:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777022.307509:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777022.307511:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775777022.307513:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777022.307516:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775777022.307518:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775777022.307521:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775777022.307524:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307526:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.307527:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307529:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777022.307532:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775777022.307535:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307537:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777022.307540:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777022.307547:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777022.307550:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307553:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777022.307557:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777022.307560:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307562:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777022.307571:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c700. 00000020:00000040:2.0:1775777022.307574:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775777022.307578:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777022.307579:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307582:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777022.307585:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777022.307588:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777022.307597:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507000. 00000100:00000001:2.0:1775777022.307618:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777022.307628:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777022.307637:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777022.307641:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777022.307645:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbf2ea94] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777022.307656:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777022.307659:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777022.307662:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775777022.307664:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777022.307665:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777022.307668:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777022.307671:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18737 00000400:00000200:2.0:1775777022.307675:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777022.307677:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777022.307680:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777022.307682:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307684:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.307686:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307688:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777022.307690:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777022.307691:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775777022.307692:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777022.307694:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775777022.307696:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775777022.307698:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775777022.307711:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307714:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.307716:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307718:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777022.307720:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775777022.307723:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307725:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777022.307726:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777022.307729:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777022.307731:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307733:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777022.307736:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777022.307738:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307740:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777022.307744:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ed80. 00000020:00000040:2.0:1775777022.307746:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775777022.307748:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777022.307750:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307751:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777022.307752:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777022.307754:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777022.307759:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506000. 00000100:00000001:2.0:1775777022.307762:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777022.307766:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777022.307773:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777022.307775:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777022.307798:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006375da9b] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777022.307806:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777022.307808:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777022.307811:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775777022.307813:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777022.307814:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777022.307817:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777022.307819:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18737 00000400:00000200:2.0:1775777022.307823:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777022.307826:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777022.307828:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777022.307830:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307832:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.307834:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307836:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777022.307838:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777022.307839:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775777022.307840:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777022.307842:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775777022.307844:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775777022.307846:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775777022.307848:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307849:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777022.307851:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307853:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777022.307855:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775777022.307857:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777022.307860:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777022.307862:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777022.307864:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777022.307866:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307868:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777022.307870:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777022.307872:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307874:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777022.307877:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f800. 00000020:00000040:2.0:1775777022.307880:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775777022.307882:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777022.307883:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777022.307885:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777022.307886:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777022.307888:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777022.307891:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504800. 00000100:00000001:2.0:1775777022.307894:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777022.307899:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777022.307910:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777022.307911:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777022.307915:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a8818441] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777022.307922:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777022.307926:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18747) 00000100:00000001:0.0:1775777022.310165:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777022.310172:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777022.310175:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777022.310189:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310205:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777022.310207:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.310211:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777022.310215:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777022.310228:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777022.310232:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824128:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777022.310238:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777022.310243:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777022.310245:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.310248:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777022.310256:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777022.310258:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777022.310266:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777022.310271:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777022.310279:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777022.310286:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099824128, portal 10 00000100:00000001:0.0:1775777022.310288:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777022.310290:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288712448 : -114841420839168 : ffff978d65e7c700) 00000100:00000040:0.0:1775777022.310297:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310303:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777022.310305:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777022.310308:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777022.310311:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099824128, offset 0 00000400:00000200:0.0:1775777022.310316:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777022.310324:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.310332:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777022.310337:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777022.310341:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777022.310345:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777022.310348:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777022.310352:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4669:4669:256:4294967295] 192.168.201.139@tcp LPNI seq info [4669:4669:8:4294967295] 00000400:00000200:0.0:1775777022.310356:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.310366:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777022.310371:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.310376:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777022.310383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777022.310386:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775777022.310391:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777022.310394:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775777022.310398:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.310401:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777022.310408:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777022.310412:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777022.310417:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777022.310419:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777022.310423:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777022.310426:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310435:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.310438:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310444:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.310445:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777022.310450:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777022.310455:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777022.310458:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777022.310461:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313410063 00000100:00000001:0.0:1775777022.310464:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313411321 00000100:00000001:0.0:1775777022.310467:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313410063 00000100:00000001:0.0:1775777022.310468:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313411321 00000100:00000001:0.0:1775777022.310471:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777022.310473:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.310476:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777022.310479:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777022/real 1775777022] req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777022.310487:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777022.310488:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777022.310490:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777022.310493:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777022.310494:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777022.310496:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777022.310499:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777022.310507:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310512:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777022.310513:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.310516:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.310517:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777022.310520:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310525:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777022.310529:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310536:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777022.310538:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.310539:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.310541:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.310543:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777022.310545:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777022.310548:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777022.310553:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777022.310555:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777022.310559:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777022.310567:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824128:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777022.310572:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777022.310575:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310581:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777022.310582:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.310584:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777022.310586:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777022.310588:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777022.310591:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824256:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777022.310594:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777022.310596:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777022.310598:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.310600:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777022.310603:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777022.310620:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777022.310624:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777022.310627:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777022.310630:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777022.310633:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099824256, portal 4 00000100:00000001:0.0:1775777022.310635:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777022.310636:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288722304 : -114841420829312 : ffff978d65e7ed80) 00000100:00000040:0.0:1775777022.310641:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310647:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777022.310649:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777022.310651:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777022.310653:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099824256, offset 0 00000400:00000200:0.0:1775777022.310656:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777022.310661:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.310664:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777022.310666:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777022.310669:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777022.310671:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777022.310673:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777022.310675:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4670:4670:256:4294967295] 192.168.201.139@tcp LPNI seq info [4670:4670:8:4294967295] 00000400:00000200:0.0:1775777022.310679:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.310684:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777022.310688:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.310691:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777022.310694:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777022.310696:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775777022.310699:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777022.310712:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775777022.310716:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.310718:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777022.310720:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777022.310723:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777022.310726:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777022.310728:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777022.310730:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777022.310734:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310741:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.310743:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310748:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.310749:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777022.310753:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777022.310756:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777022.310758:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777022.310760:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313410072 00000100:00000001:0.0:1775777022.310762:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313411321 00000100:00000001:0.0:1775777022.310764:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313410072 00000100:00000001:0.0:1775777022.310765:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313411321 00000100:00000001:0.0:1775777022.310767:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777022.310769:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.310770:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777022.310773:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777022/real 1775777022] req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777022.310813:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777022.310815:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777022.310817:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777022.310819:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777022.310821:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777022.310823:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777022.310826:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777022.310833:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310839:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777022.310840:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.310842:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.310843:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777022.310846:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310853:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777022.310856:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310863:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777022.310865:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.310866:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.310868:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.310870:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777022.310872:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777022.310874:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777022.310878:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777022.310880:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777022.310884:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777022.310891:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824256:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777022.310894:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777022.310898:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310903:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777022.310904:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.310907:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777022.310908:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777022.310910:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777022.310912:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824384:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777022.310915:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777022.310917:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777022.310918:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777022.310920:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777022.310923:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775777022.310925:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777022.310929:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777022.310932:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777022.310934:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777022.310937:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099824384, portal 4 00000100:00000001:0.0:1775777022.310939:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777022.310940:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724992 : -114841420826624 : ffff978d65e7f800) 00000100:00000040:0.0:1775777022.310946:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.310951:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777022.310953:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777022.310956:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777022.310958:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099824384, offset 0 00000400:00000200:0.0:1775777022.310961:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777022.310965:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.310968:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777022.310971:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777022.310973:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777022.310975:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777022.310977:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777022.310980:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4671:4671:256:4294967295] 192.168.201.139@tcp LPNI seq info [4671:4671:8:4294967295] 00000400:00000200:0.0:1775777022.310984:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777022.310990:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777022.310994:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.310997:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777022.311001:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777022.311003:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775777022.311007:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777022.311009:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775777022.311013:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777022.311014:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777022.311017:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777022.311020:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777022.311023:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777022.311024:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777022.311027:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777022.311030:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.311037:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.311039:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.311044:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.311046:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777022.311049:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777022.311052:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777022.311054:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777022.311056:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313410064 00000100:00000001:0.0:1775777022.311058:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313411321 00000100:00000001:0.0:1775777022.311060:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313410064 00000100:00000001:0.0:1775777022.311062:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313411321 00000100:00000001:0.0:1775777022.311064:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777022.311066:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.311068:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777022.311071:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777022/real 1775777022] req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777022.311078:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777022.311080:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777022.311081:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777022.311083:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777022.311084:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777022.311086:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777022.311089:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777022.311095:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.311105:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777022.311107:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.311108:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.311110:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777022.311113:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.311119:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777022.311122:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.311127:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777022.311129:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.311131:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777022.311132:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.311133:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777022.311135:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777022.311137:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777022.311140:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777022.311142:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777022.311144:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777022.311151:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824384:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777022.311154:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.311156:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.311158:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c700 x1862033099824128/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.311164:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777022.311166:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777022.311169:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777022.311171:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777022.311173:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777022.311175:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777022.311176:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777022.311179:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507000. 02000000:00000001:0.0:1775777022.311182:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777022.311184:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777022.311186:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c700. 00000100:00000001:0.0:1775777022.311189:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777022.311190:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.311191:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.311194:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ed80 x1862033099824256/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.311200:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777022.311201:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777022.311202:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777022.311205:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777022.311206:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777022.311207:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777022.311209:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777022.311210:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506000. 02000000:00000001:0.0:1775777022.311212:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777022.311213:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777022.311214:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ed80. 00000100:00000001:0.0:1775777022.311216:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777022.311217:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.311218:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777022.311220:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f800 x1862033099824384/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777022.311226:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777022.311227:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777022.311228:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775777022.311230:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777022.311231:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777022.311232:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777022.311234:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777022.311235:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504800. 02000000:00000001:0.0:1775777022.311236:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777022.311237:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777022.311238:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f800. 00000100:00000001:0.0:1775777022.311240:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777022.311241:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.311242:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777022.311249:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777022.311250:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777022.311252:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777022.311254:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777022.491920:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777022.491928:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777022.491930:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777022.491936:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777022.491940:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777022.491943:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777022.491946:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777022.491948:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777022.491951:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777023.004060:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777023.004067:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777023.004069:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777023.004074:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777023.004078:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777023.004083:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777023.004085:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777023.004088:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777023.004090:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777023.068648:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777023.068654:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777023.068660:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777023.068661:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777023.068664:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777023.068665:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777023.068668:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777023.068669:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777023.069022:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777023.069026:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777023.069030:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777023.518611:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777023.518619:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777023.518620:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777023.518626:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777023.518630:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777023.518634:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777023.518636:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777023.518639:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777023.518641:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777023.580038:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313411321, at 4313411328 00000100:00000010:0.0:1775777023.580061:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775777023.580194:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313411321, at 4313411328 00000100:00000010:1.0:1775777023.580200:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237420. 00000100:00000001:3.0:1775777024.029751:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777024.029759:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777024.029761:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777024.029767:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777024.029771:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777024.029800:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777024.029803:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777024.029805:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777024.029808:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777024.092084:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777024.092089:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777024.092101:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777024.092103:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777024.092106:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777024.092107:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777024.092110:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777024.092111:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777024.092281:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777024.092285:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777024.092288:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777024.541066:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777024.541072:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777024.541074:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777024.541079:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777024.541083:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777024.541086:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777024.541089:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777024.541091:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777024.541095:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777025.052330:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777025.052336:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777025.052338:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777025.052343:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777025.052347:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777025.052351:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777025.052353:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777025.052356:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777025.052359:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777025.115914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777025.115920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777025.115926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777025.115927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777025.115931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777025.115932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777025.115935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777025.115936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777025.116083:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777025.116087:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777025.116090:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777025.563858:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777025.563866:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777025.563869:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777025.563875:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777025.563879:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777025.563883:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777025.563885:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777025.563888:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777025.563890:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777026.076298:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777026.076307:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777026.076310:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777026.076315:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777026.076319:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777026.076323:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777026.076326:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777026.076328:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777026.076331:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777026.139936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777026.139943:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777026.139950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777026.139952:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777026.139956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777026.139958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777026.139961:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777026.139962:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777026.140064:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777026.140068:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777026.140072:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777026.588039:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777026.588045:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777026.588047:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777026.588053:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777026.588057:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777026.588061:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777026.588063:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777026.588066:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777026.588068:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777027.099991:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777027.099998:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777027.100000:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777027.100005:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777027.100009:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777027.100014:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777027.100016:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777027.100019:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777027.100022:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777027.164403:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777027.164409:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777027.164418:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777027.164419:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777027.164423:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777027.164424:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777027.164427:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777027.164428:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777027.164598:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777027.164602:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777027.164605:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775777027.420003:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777027.420013:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775777027.420016:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777027.420020:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777027.420024:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777027.420032:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18742 00000400:00000200:2.0:1775777027.420042:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777027.420048:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777027.420051:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777027.420054:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.420057:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.420060:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.420062:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777027.420067:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777027.420068:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775777027.420071:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777027.420074:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775777027.420076:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775777027.420078:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775777027.420081:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.420083:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.420085:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.420087:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777027.420089:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775777027.420092:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777027.420094:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777027.420097:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777027.420101:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777027.420103:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.420106:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775777027.420108:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777027.420119:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d880. 00000020:00000040:2.0:1775777027.420123:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775777027.420127:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777027.420128:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.420131:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777027.420134:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777027.420138:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777027.420146:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:2.0:1775777027.420153:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777027.420162:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777027.420171:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777027.420175:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777027.420179:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000f88f35f] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.420255:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777027.420259:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777027.420261:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777027.420267:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.420274:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777027.420276:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.420280:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777027.420282:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777027.420284:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777027.420287:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099824512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777027.420291:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777027.420294:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777027.420297:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.420299:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777027.420304:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777027.420307:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777027.420312:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777027.420316:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777027.420323:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777027.420330:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099824512, portal 25 00000100:00000001:0.0:1775777027.420333:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777027.420334:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716928 : -114841420834688 : ffff978d65e7d880) 00000100:00000040:0.0:1775777027.420339:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.420346:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777027.420349:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777027.420351:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777027.420354:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099824512, offset 0 00000400:00000200:0.0:1775777027.420360:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777027.420367:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.420373:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777027.420378:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777027.420382:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777027.420385:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777027.420388:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777027.420393:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4672:4672:256:4294967295] 192.168.201.139@tcp LPNI seq info [4672:4672:8:4294967295] 00000400:00000200:0.0:1775777027.420396:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.420407:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777027.420412:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.420417:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777027.420424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777027.420427:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775777027.420432:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777027.420435:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775777027.420439:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.420441:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777027.420449:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777027.420454:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777027.420459:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777027.420462:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777027.420465:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777027.420559:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.422000:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.422015:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.422025:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.422027:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777027.422040:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777027.422047:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777027.422050:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313416439 00000100:00000010:0.0:1775777027.422058:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777027.422065:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777027.422072:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313416439 00000100:00000010:0.0:1775777027.422074:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f2375a0. 00000100:00000001:0.0:1775777027.422077:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777027.422079:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777027.422082:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.422085:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777027.422089:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777027/real 1775777027] req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777027.422100:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777027.422101:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777027.422104:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777027.422108:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777027.422110:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777027.422113:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777027.422117:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777027.422131:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.422137:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777027.422139:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.422141:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.422143:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777027.422147:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.422153:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777027.422158:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.422166:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777027.422169:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.422170:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.422172:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.422174:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777027.422177:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777027.422180:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777027.422184:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777027.422187:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777027.422192:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777027.422201:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099824512:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777027.422206:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.422208:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.422210:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d880 x1862033099824512/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.422223:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777027.422226:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777027.422229:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777027.422234:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777027.422236:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777027.422238:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777027.422240:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777027.422242:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:0.0:1775777027.422246:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777027.422248:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777027.422249:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d880. 00000100:00000001:0.0:1775777027.422253:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777027.422254:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.422256:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.422263:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777027.422265:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.422267:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777027.422269:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.423853:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777027.423862:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777027.423867:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775777027.423869:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777027.423873:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777027.423877:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777027.423884:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18742 00000400:00000200:2.0:1775777027.423893:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777027.423899:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777027.423902:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777027.423905:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.423908:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.423910:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.423912:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777027.423916:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777027.423918:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775777027.423920:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777027.423923:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775777027.423926:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775777027.423928:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775777027.423930:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.423932:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.423934:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.423936:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777027.423939:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775777027.423942:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777027.423944:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777027.423946:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777027.423954:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777027.423957:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.423959:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777027.423963:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777027.423966:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.423968:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777027.423977:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7dc00. 00000020:00000040:2.0:1775777027.423980:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775777027.423984:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777027.423986:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.423989:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777027.423992:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777027.423996:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777027.424005:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506c00. 00000100:00000001:2.0:1775777027.424011:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777027.424020:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777027.424030:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777027.424035:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777027.424039:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f1188f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777027.424073:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777027.424076:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777027.424079:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775777027.424080:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777027.424081:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777027.424084:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777027.424086:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18742 00000400:00000200:2.0:1775777027.424090:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777027.424093:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777027.424095:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777027.424097:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424099:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.424101:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424102:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777027.424104:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777027.424116:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775777027.424117:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777027.424119:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775777027.424121:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775777027.424123:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775777027.424126:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424128:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.424130:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424131:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777027.424134:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775777027.424136:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424138:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777027.424140:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777027.424143:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777027.424145:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424147:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777027.424149:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777027.424151:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424152:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777027.424156:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7df80. 00000020:00000040:2.0:1775777027.424158:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775777027.424161:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777027.424162:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424163:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777027.424164:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777027.424166:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777027.424171:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505c00. 00000100:00000001:2.0:1775777027.424176:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777027.424181:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777027.424190:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777027.424192:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777027.424195:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009deec8f8] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777027.424202:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777027.424205:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777027.424208:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775777027.424209:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777027.424210:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777027.424212:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777027.424215:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18742 00000400:00000200:2.0:1775777027.424219:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777027.424222:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777027.424224:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777027.424226:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424228:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.424230:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424232:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777027.424233:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777027.424235:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775777027.424236:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777027.424238:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775777027.424239:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775777027.424241:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775777027.424244:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424246:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.424248:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424249:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777027.424251:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775777027.424254:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424256:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777027.424257:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777027.424260:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777027.424263:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424265:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777027.424268:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777027.424270:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424271:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777027.424275:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c380. 00000020:00000040:2.0:1775777027.424277:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775777027.424279:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777027.424280:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424282:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777027.424283:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777027.424284:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777027.424288:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506400. 00000100:00000001:2.0:1775777027.424291:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777027.424294:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777027.424300:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777027.424302:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777027.424304:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f231afe] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777027.424310:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777027.424312:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777027.424315:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775777027.424317:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777027.424318:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777027.424321:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777027.424322:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18742 00000400:00000200:2.0:1775777027.424326:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777027.424329:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777027.424331:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777027.424333:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424335:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.424336:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424338:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777027.424340:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777027.424341:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775777027.424343:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777027.424344:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775777027.424346:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775777027.424348:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775777027.424350:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424352:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777027.424353:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424355:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777027.424357:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775777027.424359:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777027.424360:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777027.424362:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777027.424364:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777027.424366:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424368:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777027.424369:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777027.424372:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424373:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777027.424376:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f480. 00000020:00000040:2.0:1775777027.424377:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775777027.424379:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777027.424380:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777027.424382:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777027.424383:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777027.424384:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777027.424387:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505800. 00000100:00000001:2.0:1775777027.424389:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777027.424393:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777027.424404:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777027.424406:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777027.424409:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002c91b8b4] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777027.424416:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777027.424421:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18752) 00000100:00000001:0.0:1775777027.424910:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777027.424914:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777027.424916:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777027.424923:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.424930:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777027.424932:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.424935:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777027.424937:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777027.424940:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777027.424943:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777027.424947:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777027.424951:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777027.424953:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.424955:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777027.424961:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777027.424964:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777027.424969:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777027.424973:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777027.424978:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777027.424986:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099824640, portal 10 00000100:00000001:0.0:1775777027.424988:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777027.424990:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288717824 : -114841420833792 : ffff978d65e7dc00) 00000100:00000040:0.0:1775777027.424997:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425004:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777027.425007:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777027.425009:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777027.425012:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099824640, offset 0 00000400:00000200:0.0:1775777027.425018:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777027.425026:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.425031:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777027.425036:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777027.425040:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777027.425043:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777027.425046:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777027.425049:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4673:4673:256:4294967295] 192.168.201.139@tcp LPNI seq info [4673:4673:8:4294967295] 00000400:00000200:0.0:1775777027.425053:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.425062:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777027.425067:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.425072:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777027.425079:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777027.425082:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775777027.425087:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777027.425090:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775777027.425094:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.425096:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777027.425103:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777027.425107:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777027.425112:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777027.425115:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777027.425119:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777027.425122:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425130:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.425133:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425139:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.425141:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777027.425146:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777027.425151:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777027.425154:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777027.425157:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313415177 00000100:00000001:0.0:1775777027.425160:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313416439 00000100:00000001:0.0:1775777027.425162:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313415177 00000100:00000001:0.0:1775777027.425164:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313416439 00000100:00000001:0.0:1775777027.425166:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777027.425168:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.425170:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777027.425173:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777027/real 1775777027] req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777027.425182:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777027.425183:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777027.425185:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777027.425186:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777027.425188:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777027.425190:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777027.425193:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777027.425206:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425212:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777027.425213:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.425215:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.425216:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777027.425220:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425226:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777027.425230:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425236:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777027.425238:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.425240:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.425241:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.425243:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777027.425244:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777027.425247:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777027.425251:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777027.425253:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777027.425256:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777027.425264:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824640:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777027.425268:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777027.425271:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425283:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777027.425285:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.425287:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777027.425288:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777027.425291:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777027.425293:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777027.425295:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777027.425297:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777027.425298:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.425300:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777027.425303:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777027.425304:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777027.425308:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777027.425310:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777027.425313:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777027.425315:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099824768, portal 10 00000100:00000001:0.0:1775777027.425317:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777027.425318:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288718720 : -114841420832896 : ffff978d65e7df80) 00000100:00000040:0.0:1775777027.425322:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425328:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777027.425330:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777027.425332:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777027.425334:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099824768, offset 0 00000400:00000200:0.0:1775777027.425337:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777027.425342:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.425345:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777027.425347:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777027.425350:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777027.425352:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777027.425354:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777027.425356:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4674:4674:256:4294967295] 192.168.201.139@tcp LPNI seq info [4674:4674:8:4294967295] 00000400:00000200:0.0:1775777027.425360:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.425365:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777027.425369:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.425372:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777027.425376:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777027.425379:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775777027.425383:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777027.425385:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775777027.425388:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.425390:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777027.425392:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777027.425395:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777027.425398:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777027.425400:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777027.425402:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777027.425405:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425411:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.425414:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425419:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.425421:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777027.425424:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777027.425427:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777027.425430:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777027.425431:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313415178 00000100:00000001:0.0:1775777027.425433:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313416439 00000100:00000001:0.0:1775777027.425435:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313415178 00000100:00000001:0.0:1775777027.425438:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313416439 00000100:00000001:0.0:1775777027.425439:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777027.425442:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.425444:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777027.425447:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777027/real 1775777027] req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777027.425455:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777027.425456:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777027.425457:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777027.425460:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777027.425461:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777027.425463:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777027.425478:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777027.425485:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425491:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777027.425492:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.425494:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.425495:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777027.425498:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425504:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777027.425507:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425514:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777027.425516:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.425517:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.425519:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.425520:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777027.425522:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777027.425524:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777027.425528:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777027.425530:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777027.425534:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777027.425542:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824768:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777027.425545:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777027.425548:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425555:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777027.425556:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.425559:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777027.425560:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777027.425562:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777027.425565:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777027.425568:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777027.425569:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777027.425571:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.425573:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777027.425576:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777027.425578:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777027.425582:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777027.425584:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777027.425587:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777027.425590:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099824896, portal 4 00000100:00000001:0.0:1775777027.425591:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777027.425592:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288711552 : -114841420840064 : ffff978d65e7c380) 00000100:00000040:0.0:1775777027.425598:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425622:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777027.425625:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777027.425628:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777027.425630:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099824896, offset 0 00000400:00000200:0.0:1775777027.425634:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777027.425638:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.425641:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777027.425644:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777027.425646:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777027.425648:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777027.425650:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777027.425653:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4675:4675:256:4294967295] 192.168.201.139@tcp LPNI seq info [4675:4675:8:4294967295] 00000400:00000200:0.0:1775777027.425656:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.425662:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777027.425665:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.425668:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777027.425672:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777027.425674:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775777027.425678:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777027.425681:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775777027.425684:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.425686:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777027.425689:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777027.425692:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777027.425695:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777027.425696:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777027.425699:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777027.425702:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.425709:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.425712:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430359:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.430364:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777027.430382:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777027.430387:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777027.430396:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777027.430400:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313415192 00000100:00000001:0.0:1775777027.430405:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313416439 00000100:00000001:0.0:1775777027.430407:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313415192 00000100:00000001:0.0:1775777027.430410:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313416439 00000100:00000001:0.0:1775777027.430412:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777027.430416:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.430419:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777027.430430:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777027/real 1775777027] req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777027.430444:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777027.430446:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777027.430449:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777027.430452:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777027.430455:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777027.430458:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777027.430462:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777027.430471:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430480:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777027.430481:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.430484:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.430485:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777027.430489:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430495:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777027.430500:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430508:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777027.430511:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.430512:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.430514:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.430516:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777027.430518:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777027.430521:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777027.430526:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777027.430529:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777027.430533:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777027.430541:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099824896:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777027.430546:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777027.430550:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430557:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777027.430559:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.430562:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777027.430566:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777027.430569:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777027.430572:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777027.430575:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777027.430579:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777027.430582:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777027.430584:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777027.430594:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759cc00. 02000000:00000001:0.0:1775777027.430596:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777027.430641:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777027.430646:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777027.430651:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777027.430657:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099825024, portal 4 00000100:00000001:0.0:1775777027.430659:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777027.430661:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288724096 : -114841420827520 : ffff978d65e7f480) 00000100:00000040:0.0:1775777027.430669:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430676:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777027.430679:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777027.430682:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777027.430684:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099825024, offset 0 00000400:00000200:0.0:1775777027.430690:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777027.430697:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.430703:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777027.430708:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777027.430712:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777027.430716:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777027.430719:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777027.430723:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4676:4676:256:4294967295] 192.168.201.139@tcp LPNI seq info [4676:4676:8:4294967295] 00000400:00000200:0.0:1775777027.430728:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777027.430738:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777027.430743:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.430748:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777027.430755:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777027.430759:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775777027.430765:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777027.430769:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775777027.430773:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777027.430775:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777027.430799:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777027.430804:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777027.430809:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777027.430810:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777027.430814:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777027.430817:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430826:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.430829:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430835:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.430837:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777027.430840:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777027.430843:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777027.430846:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777027.430848:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313415184 00000100:00000001:0.0:1775777027.430851:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313416439 00000100:00000001:0.0:1775777027.430853:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313415184 00000100:00000001:0.0:1775777027.430855:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313416439 00000100:00000001:0.0:1775777027.430857:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777027.430859:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.430860:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777027.430864:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777027/real 1775777027] req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777027.430871:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777027.430872:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777027.430874:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777027.430876:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777027.430878:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777027.430880:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777027.430883:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777027.430891:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430903:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777027.430905:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.430907:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.430908:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777027.430911:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430918:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777027.430921:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430928:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777027.430931:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.430932:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777027.430934:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.430935:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777027.430937:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777027.430940:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777027.430943:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777027.430945:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777027.430949:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777027.430956:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825024:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777027.430960:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.430963:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.430967:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7dc00 x1862033099824640/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.430974:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777027.430976:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777027.430980:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777027.430984:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777027.430987:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777027.430989:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777027.430991:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777027.430994:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:0.0:1775777027.430998:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777027.431000:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777027.431002:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7dc00. 00000100:00000001:0.0:1775777027.431006:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777027.431007:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.431009:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.431013:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7df80 x1862033099824768/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.431020:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777027.431022:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777027.431023:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777027.431026:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777027.431027:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777027.431028:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777027.431030:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777027.431032:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505c00. 02000000:00000001:0.0:1775777027.431035:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777027.431036:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777027.431037:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7df80. 00000100:00000001:0.0:1775777027.431039:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777027.431040:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.431042:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.431045:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c380 x1862033099824896/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.431051:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777027.431053:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777027.431054:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777027.431056:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777027.431057:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777027.431058:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777027.431061:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777027.431062:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506400. 02000000:00000001:0.0:1775777027.431064:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777027.431065:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777027.431066:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c380. 00000100:00000001:0.0:1775777027.431068:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777027.431070:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.431071:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777027.431074:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f480 x1862033099825024/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777027.431081:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777027.431082:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777027.431083:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759cc00. 02000000:00000001:0.0:1775777027.431085:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777027.431086:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777027.431088:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777027.431090:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777027.431091:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505800. 02000000:00000001:0.0:1775777027.431093:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777027.436554:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777027.436564:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f480. 00000100:00000001:0.0:1775777027.436574:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777027.436576:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.436580:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777027.436588:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777027.436590:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777027.436593:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777027.436595:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777027.611926:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777027.611934:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777027.611936:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777027.611944:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777027.611949:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777027.611954:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777027.611956:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777027.611959:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777027.611962:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777028.126298:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777028.126305:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777028.126307:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777028.126313:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777028.126318:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777028.126323:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777028.126326:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777028.126328:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777028.126331:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777028.188318:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777028.188324:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777028.188329:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777028.188331:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777028.188334:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777028.188335:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777028.188338:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777028.188339:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777028.190322:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777028.190328:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777028.190333:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777028.635935:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777028.635942:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777028.635944:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777028.635951:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777028.635959:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777028.635964:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777028.635966:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777028.635969:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777028.635972:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777028.700169:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313416439, at 4313416448 00000100:00000010:0.0:1775777028.700192:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775777028.700634:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313416439, at 4313416448 00000100:00000010:1.0:1775777028.700641:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f2375a0. 00000100:00000001:3.0:1775777029.147908:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777029.147914:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777029.147916:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777029.147921:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777029.147925:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777029.147929:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777029.147932:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777029.147934:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777029.147937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777029.214208:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777029.214228:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777029.214231:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777029.214235:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777029.214239:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777029.214242:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777029.214244:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777029.214246:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777029.214250:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777029.214252:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777029.214255:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777029.214256:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777029.214260:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777029.214261:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777029.214428:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777029.214432:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777029.214435:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777029.662413:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777029.662421:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777029.662423:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777029.662428:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777029.662432:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777029.662436:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777029.662438:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777029.662441:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777029.662444:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777030.172340:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777030.172347:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777030.172350:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777030.172356:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777030.172360:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777030.172364:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777030.172367:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777030.172370:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777030.172373:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777030.236058:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777030.236064:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777030.236070:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777030.236071:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777030.236074:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777030.236076:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777030.236079:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777030.236081:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777030.236415:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777030.236418:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777030.236421:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777030.685981:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777030.685988:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777030.685991:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777030.685997:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777030.686001:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777030.686006:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777030.686008:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777030.686011:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777030.686013:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777031.196871:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777031.196878:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777031.196879:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777031.196885:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777031.196889:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777031.196893:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777031.196896:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777031.196898:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777031.196901:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777031.260407:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777031.260414:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777031.260420:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777031.260421:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777031.260424:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777031.260425:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777031.260440:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777031.260441:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777031.260745:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777031.260748:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777031.260752:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775777031.708161:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777031.708167:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777031.708169:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777031.708175:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777031.708179:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777031.708182:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777031.708186:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777031.708188:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777031.708191:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.219922:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777032.219940:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777032.219943:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777032.219949:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777032.219953:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777032.219957:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777032.219959:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777032.219967:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777032.219971:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284012:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777032.284019:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284026:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777032.284029:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777032.284031:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777032.284035:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284038:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777032.284040:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284042:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284044:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284342:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777032.284344:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777032.284346:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777032.284347:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284349:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777032.284350:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284352:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284353:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284357:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777032.284358:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777032.284359:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777032.284361:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284362:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777032.284363:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284365:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777032.284366:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777032.289362:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777032.289368:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777032.289372:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775777032.540087:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777032.540098:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775777032.540100:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777032.540104:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777032.540109:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777032.540117:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18747 00000400:00000200:3.0:1775777032.540139:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777032.540147:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777032.540150:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777032.540153:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540156:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540158:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540161:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777032.540164:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777032.540166:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775777032.540169:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777032.540171:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775777032.540174:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775777032.540176:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775777032.540179:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540181:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540182:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540184:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777032.540186:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775777032.540190:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540192:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777032.540195:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777032.540199:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777032.540201:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540203:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775777032.540205:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777032.540227:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775777032.540232:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775777032.540235:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777032.540237:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540240:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777032.540242:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777032.540247:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777032.540255:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775777032.540262:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777032.540270:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777032.540281:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777032.540286:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777032.540290:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777032.540315:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777032.540319:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777032.540322:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775777032.540324:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777032.540325:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777032.540328:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777032.540331:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18747 00000400:00000200:3.0:1775777032.540335:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777032.540338:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777032.540340:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777032.540343:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540345:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540348:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540350:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777032.540352:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777032.540353:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775777032.540354:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777032.540356:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775777032.540358:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775777032.540360:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775777032.540362:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540364:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540366:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540367:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777032.540369:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775777032.540372:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540374:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777032.540376:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777032.540381:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777032.540383:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540384:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777032.540388:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777032.540391:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540392:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777032.540396:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775777032.540398:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775777032.540400:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777032.540402:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540404:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777032.540405:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777032.540407:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777032.540411:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000001:3.0:1775777032.540415:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.540420:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1775777032.540420:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777032.540426:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777032.540430:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775777032.540430:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777032.540433:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777032.540440:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775777032.540441:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777032.540461:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.540462:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775777032.540464:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775777032.540465:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777032.540470:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:0.0:1775777032.540471:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775777032.540472:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1775777032.540475:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1775777032.540476:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1775777032.540480:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540481:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777032.540485:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18747 00000100:00100000:0.0:1775777032.540488:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099825152:192.168.201.139@tcp:250:kworker.0 00000400:00000200:3.0:1775777032.540492:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777032.540495:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777032.540497:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777032.540501:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777032.540505:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775777032.540505:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540509:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 02000000:00000001:0.0:1775777032.540510:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775777032.540511:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:0.0:1775777032.540515:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775777032.540515:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777032.540517:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 02000000:00000001:0.0:1775777032.540520:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1775777032.540520:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000010:0.0:1775777032.540535:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777032.540537:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775777032.540538:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775777032.540542:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000400:00000200:0.0:1775777032.540546:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000001:3.0:1775777032.540546:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775777032.540552:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000010:0.0:1775777032.540553:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000100:00000001:3.0:1775777032.540555:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540559:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540561:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000010:0.0:1775777032.540562:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000020:00000040:3.0:1775777032.540563:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775777032.540568:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000200:0.0:1775777032.540572:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099825152, portal 25 00000100:00000001:3.0:1775777032.540572:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777032.540576:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775777032.540578:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1775777032.540581:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777032.540584:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540586:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777032.540588:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775777032.540590:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751255296 : -114841958296320 : ffff978d45ded500) 00000008:00000001:3.0:1775777032.540591:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540596:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777032.540601:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775777032.540601:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775777032.540606:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775777032.540611:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777032.540614:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.540616:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775777032.540617:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777032.540620:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1775777032.540623:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 02000000:00000001:3.0:1775777032.540624:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777032.540629:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775777032.540632:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775777032.540636:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00100000:3.0:1775777032.540638:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775777032.540643:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099825152, offset 0 00000100:00000001:3.0:1775777032.540651:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777032.540655:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.540656:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000040:3.0:1775777032.540661:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775777032.540667:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777032.540675:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.540677:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00080000:3.0:1775777032.540680:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775777032.540683:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775777032.540686:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775777032.540688:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777032.540690:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777032.540693:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777032.540697:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18747 00000400:00000200:3.0:1775777032.540701:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777032.540705:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777032.540707:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777032.540709:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540712:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540714:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540716:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777032.540718:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777032.540720:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775777032.540721:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777032.540723:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775777032.540725:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775777032.540727:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775777032.540730:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540732:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540734:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540735:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777032.540738:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775777032.540740:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540742:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777032.540744:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777032.540748:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777032.540749:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540751:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777032.540753:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777032.540756:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540757:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777032.540760:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775777032.540762:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775777032.540765:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777032.540766:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540768:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777032.540769:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777032.540770:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777032.540775:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775777032.540819:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.540824:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000100:00100000:3.0:1775777032.540826:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775777032.540831:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777032.540837:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540842:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775777032.540845:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4677:4677:256:4294967295] 192.168.201.139@tcp LPNI seq info [4677:4677:8:4294967295] 00000100:00000001:3.0:1775777032.540846:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777032.540854:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775777032.540855:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777032.540864:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.540870:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00080000:3.0:1775777032.540870:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777032.540878:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775777032.540880:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:0.0:1775777032.540884:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000100:00080000:3.0:1775777032.540884:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777032.540889:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777032.540892:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18747 00000400:00000200:3.0:1775777032.540896:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777032.540899:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777032.540901:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777032.540904:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540906:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540908:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540910:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777032.540912:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777032.540913:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775777032.540915:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777032.540916:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775777032.540919:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775777032.540920:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775777032.540923:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540924:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777032.540926:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540927:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777032.540929:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775777032.540932:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777032.540933:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777032.540935:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777032.540938:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777032.540940:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540942:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777032.540944:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777032.540946:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540948:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777032.540952:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775777032.540955:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775777032.540957:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777032.540958:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777032.540959:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777032.540961:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777032.540962:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775777032.540968:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 02000000:00000010:3.0:1775777032.540968:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775777032.540973:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775777032.540979:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00100000:3.0:1775777032.540980:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1775777032.540985:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1775777032.540991:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777032.540994:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775777032.540997:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000040:3.0:1775777032.541001:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775777032.541002:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1775777032.541008:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777032.541013:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18757) 00000800:00000100:0.0:1775777032.541026:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.541030:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777032.541038:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777032.541044:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777032.541049:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777032.541051:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777032.541056:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777032.541061:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541071:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.541074:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541082:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.541084:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777032.541089:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.541093:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777032.541095:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313421558 00000100:00000010:0.0:1775777032.541100:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777032.541106:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777032.541112:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313421558 00000100:00000010:0.0:1775777032.541116:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f2376c0. 00000100:00000001:0.0:1775777032.541119:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777032.541122:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777032.541124:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.541127:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777032.541131:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777032/real 1775777032] req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777032.541140:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777032.541141:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777032.541143:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777032.541146:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777032.541147:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777032.541149:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777032.541152:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777032.541161:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541167:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777032.541168:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541170:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.541172:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777032.541175:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541181:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777032.541185:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541191:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777032.541194:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541195:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.541196:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541199:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777032.541201:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777032.541203:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777032.541206:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777032.541208:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777032.541212:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777032.541218:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099825152:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777032.541222:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541224:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.541227:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded500 x1862033099825152/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541232:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777032.541234:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777032.541236:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777032.541240:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777032.541241:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777032.541243:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777032.541245:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777032.541247:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a000. 02000000:00000001:0.0:1775777032.541250:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777032.541252:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777032.541253:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded500. 00000100:00000001:0.0:1775777032.541256:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777032.541257:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541259:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541265:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777032.541267:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.541269:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777032.541270:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777032.541271:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777032.541274:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541285:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777032.541286:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777032.541289:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777032.541290:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777032.541292:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777032.541295:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825280:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777032.541297:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777032.541300:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777032.541301:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777032.541303:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777032.541306:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777032.541308:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.541311:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777032.541314:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777032.541317:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777032.541320:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099825280, portal 10 00000100:00000001:0.0:1775777032.541322:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777032.541323:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751258880 : -114841958292736 : ffff978d45dee300) 00000100:00000040:0.0:1775777032.541330:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541335:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777032.541337:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777032.541339:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777032.541342:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099825280, offset 0 00000400:00000200:0.0:1775777032.541345:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777032.541349:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777032.541353:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777032.541355:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777032.541357:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777032.541360:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777032.541362:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777032.541365:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4678:4678:256:4294967295] 192.168.201.139@tcp LPNI seq info [4678:4678:8:4294967295] 00000400:00000200:0.0:1775777032.541369:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777032.541374:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777032.541378:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.541381:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777032.541385:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777032.541387:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775777032.541390:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777032.541392:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775777032.541395:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.541397:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777032.541399:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777032.541402:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777032.541405:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777032.541407:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777032.541409:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777032.541412:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541418:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.541421:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541426:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.541427:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777032.541430:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777032.541433:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777032.541436:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777032.541438:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313420293 00000100:00000001:0.0:1775777032.541440:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313421558 00000100:00000001:0.0:1775777032.541442:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313420293 00000100:00000001:0.0:1775777032.541444:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313421558 00000100:00000001:0.0:1775777032.541446:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777032.541448:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.541450:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777032.541453:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777032/real 1775777032] req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777032.541459:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777032.541461:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777032.541462:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777032.541464:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777032.541466:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777032.541467:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777032.541470:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777032.541477:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541482:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777032.541483:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541485:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.541486:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777032.541489:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541495:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777032.541499:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541504:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777032.541506:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541508:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.541509:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.541511:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777032.541513:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777032.541516:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777032.541519:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777032.541521:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777032.541535:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777032.541543:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825280:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777032.541547:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777032.541550:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541556:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777032.541558:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777032.541560:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777032.541562:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777032.541564:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777032.541566:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825408:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777032.541569:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777032.541571:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777032.541573:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777032.541574:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777032.541577:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777032.541579:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.541582:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777032.541584:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777032.541586:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777032.541589:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099825408, portal 10 00000100:00000001:0.0:1775777032.541590:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777032.541592:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751254400 : -114841958297216 : ffff978d45ded180) 00000100:00000040:0.0:1775777032.541597:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.541603:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777032.541605:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777032.541607:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777032.541610:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099825408, offset 0 00000400:00000200:0.0:1775777032.541613:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777032.541617:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777032.541620:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777032.541622:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777032.541624:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777032.541626:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777032.541640:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777032.541644:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4679:4679:256:4294967295] 192.168.201.139@tcp LPNI seq info [4679:4679:8:4294967295] 00000400:00000200:0.0:1775777032.541648:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777032.541653:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777032.541658:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.541661:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777032.541665:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777032.541667:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775777032.541671:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777032.541673:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775777032.541677:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.541678:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777032.541682:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777032.541685:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777032.541688:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777032.544077:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777032.544087:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777032.544099:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.544112:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.544116:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.544123:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.544125:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777032.544137:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777032.544143:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777032.544149:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777032.544152:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313420297 00000100:00000001:0.0:1775777032.544156:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313421558 00000100:00000001:0.0:1775777032.544159:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313420297 00000100:00000001:0.0:1775777032.544161:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313421558 00000100:00000001:0.0:1775777032.544163:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777032.544166:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.544169:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777032.544174:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777032/real 1775777032] req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777032.544185:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777032.544187:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777032.544190:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777032.544194:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777032.544195:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777032.544198:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777032.544201:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777032.544210:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.544217:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777032.544218:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.544221:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.544223:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777032.544227:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.544234:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777032.546466:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.546502:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777032.546508:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.546511:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.546514:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.546517:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777032.546519:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777032.546533:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777032.546542:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777032.546545:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777032.546552:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777032.546561:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825408:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777032.546569:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777032.546573:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.546580:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777032.546581:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777032.546584:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777032.546588:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777032.546590:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777032.546594:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825536:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777032.546597:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777032.546602:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777032.546604:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777032.546606:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777032.546616:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777032.546618:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.546637:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777032.546643:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777032.546651:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777032.546658:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099825536, portal 4 00000100:00000001:0.0:1775777032.546660:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777032.546662:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751260672 : -114841958290944 : ffff978d45deea00) 00000100:00000040:0.0:1775777032.546669:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.546676:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777032.546679:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777032.546681:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777032.546683:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099825536, offset 0 00000400:00000200:0.0:1775777032.548043:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777032.548058:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777032.548069:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777032.548074:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777032.548079:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777032.548083:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777032.548086:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777032.548089:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4680:4680:256:4294967295] 192.168.201.139@tcp LPNI seq info [4680:4680:8:4294967295] 00000400:00000200:0.0:1775777032.548094:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777032.548104:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777032.548109:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.548118:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777032.548125:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777032.548129:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775777032.548135:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777032.548138:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775777032.548142:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.548144:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777032.548151:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777032.548156:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777032.548162:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777032.548165:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777032.548170:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777032.548176:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548187:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.548190:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548197:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.548198:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777032.548204:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777032.548209:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777032.548212:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777032.548215:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313420310 00000100:00000001:0.0:1775777032.548218:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313421558 00000100:00000001:0.0:1775777032.548221:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313420310 00000100:00000001:0.0:1775777032.548223:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313421558 00000100:00000001:0.0:1775777032.548225:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777032.548227:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.548230:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777032.548234:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777032/real 1775777032] req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777032.548244:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777032.548245:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777032.548248:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777032.548251:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777032.548253:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777032.548255:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777032.548258:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777032.548266:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548273:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777032.548275:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.548277:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.548279:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777032.548282:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548289:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777032.548293:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548299:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777032.548302:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.548303:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.548304:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.548307:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777032.548309:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777032.548313:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777032.548317:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777032.548319:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777032.548322:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777032.548329:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825536:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777032.548333:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777032.548337:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548342:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777032.548344:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777032.548347:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777032.548350:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777032.548352:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777032.548355:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825664:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777032.548358:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777032.548361:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777032.548363:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777032.548365:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777032.548369:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759cc00. 02000000:00000001:0.0:1775777032.548372:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777032.548375:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777032.548378:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777032.548382:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777032.548385:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099825664, portal 4 00000100:00000001:0.0:1775777032.548387:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777032.548388:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751264256 : -114841958287360 : ffff978d45def800) 00000100:00000040:0.0:1775777032.548394:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548399:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777032.548401:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777032.548403:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777032.548406:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099825664, offset 0 00000400:00000200:0.0:1775777032.548410:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777032.548414:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777032.548416:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777032.548419:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777032.548420:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777032.548423:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777032.548425:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777032.548427:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4681:4681:256:4294967295] 192.168.201.139@tcp LPNI seq info [4681:4681:8:4294967295] 00000400:00000200:0.0:1775777032.548431:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777032.548436:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777032.548440:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.548444:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777032.548447:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777032.548450:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775777032.548453:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777032.548455:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775777032.548459:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777032.548461:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777032.548463:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777032.548466:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777032.548469:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777032.548470:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777032.548472:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777032.548475:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548482:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.548485:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548490:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.548492:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777032.548495:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777032.548498:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777032.548501:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777032.548503:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313420301 00000100:00000001:0.0:1775777032.548505:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313421558 00000100:00000001:0.0:1775777032.548508:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313420301 00000100:00000001:0.0:1775777032.548509:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313421558 00000100:00000001:0.0:1775777032.548511:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777032.548513:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.548515:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777032.548518:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777032/real 1775777032] req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777032.548536:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777032.548537:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777032.548539:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777032.548542:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777032.548543:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777032.548545:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777032.548547:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777032.548554:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548559:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777032.548561:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.548562:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.548564:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777032.548567:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548573:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777032.548576:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548593:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777032.548596:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.548597:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777032.548598:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.548600:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777032.548602:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777032.548605:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777032.548608:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777032.548610:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777032.548615:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777032.548621:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825664:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777032.548625:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.548635:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.548640:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee300 x1862033099825280/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548646:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777032.548648:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777032.548651:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777032.548654:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777032.548657:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777032.548659:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777032.548661:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777032.548663:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158800. 02000000:00000001:0.0:1775777032.548666:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777032.548668:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777032.548669:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee300. 00000100:00000001:0.0:1775777032.548673:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777032.548674:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.548676:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.548680:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099825408/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.548686:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777032.548687:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777032.552172:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777032.552186:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777032.552189:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777032.552191:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777032.552195:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777032.552197:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158000. 02000000:00000001:0.0:1775777032.552201:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777032.552204:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777032.552207:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:0.0:1775777032.552212:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777032.552214:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.552218:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.552231:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099825536/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.552241:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777032.552243:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777032.552245:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777032.552247:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777032.552249:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777032.552250:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777032.552251:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777032.552253:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115ac00. 02000000:00000001:0.0:1775777032.552255:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777032.552257:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777032.552257:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:0.0:1775777032.552261:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777032.552262:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.552263:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777032.552268:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def800 x1862033099825664/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777032.552274:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777032.552275:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777032.552276:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759cc00. 02000000:00000001:0.0:1775777032.552278:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777032.552279:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777032.552281:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777032.552282:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777032.552283:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:0.0:1775777032.552286:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777032.552287:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777032.552288:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def800. 00000100:00000001:0.0:1775777032.552291:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777032.552292:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.552295:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777032.552307:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777032.552309:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777032.552312:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777032.552314:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777032.732124:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775777032.732131:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775777032.732133:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777032.732146:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:1.0:1775777032.732150:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775777032.732154:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777032.732157:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775777032.732159:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775777032.732162:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777033.244022:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777033.244030:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777033.244031:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777033.244037:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777033.244041:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777033.244057:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777033.244060:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777033.244062:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777033.244064:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777033.308535:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777033.308541:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777033.308546:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777033.308548:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777033.308551:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777033.308552:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777033.308556:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777033.308557:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777033.318929:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777033.318935:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777033.318939:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777033.762861:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777033.762878:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777033.762880:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777033.762887:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777033.762892:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777033.762896:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777033.762899:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777033.762902:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777033.762906:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777033.819958:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313421558, at 4313421568 00000100:00000010:1.0:1775777033.819975:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f2376c0. 00000100:00000040:0.0:1775777033.819988:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313421558, at 4313421568 00000100:00000010:0.0:1775777033.819997:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775777034.267921:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777034.267931:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777034.267932:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777034.267937:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777034.267941:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777034.267945:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777034.267947:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777034.267950:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777034.267953:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777034.332270:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777034.332277:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777034.332282:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777034.332283:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777034.332287:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777034.332288:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777034.332291:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777034.332292:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777034.332457:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777034.332461:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777034.332465:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777034.779920:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777034.779927:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777034.779929:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777034.779935:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777034.779939:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777034.779943:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777034.779945:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777034.779948:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777034.779950:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777035.294434:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777035.294441:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777035.294443:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777035.294448:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777035.294451:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777035.294455:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777035.294457:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777035.294460:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777035.294463:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777035.356282:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777035.356287:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777035.356293:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777035.356294:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777035.356298:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777035.356299:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777035.356303:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777035.356304:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777035.356396:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777035.356399:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777035.356402:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777035.806141:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777035.806148:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777035.806150:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777035.806157:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777035.806161:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777035.806165:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777035.806168:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777035.806171:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777035.806174:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777036.315957:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777036.315967:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777036.315969:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777036.315976:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777036.315982:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777036.315986:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777036.315989:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777036.315991:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777036.315994:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777036.380932:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777036.380940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777036.380947:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777036.380948:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777036.380952:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777036.380953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777036.380957:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777036.380958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777036.381201:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777036.381205:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777036.381210:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777036.827927:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777036.827936:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777036.827938:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777036.827957:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777036.827962:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777036.827966:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777036.827969:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777036.827972:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777036.827975:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777037.343366:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777037.343374:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777037.343376:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777037.343382:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777037.343385:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777037.343389:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777037.343393:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777037.343396:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777037.343398:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777037.404303:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777037.404309:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777037.404315:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777037.404316:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777037.404319:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777037.404320:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777037.404325:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777037.404326:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777037.404437:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777037.404481:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777037.404485:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775777037.660052:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777037.660063:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775777037.660066:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777037.660070:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777037.660075:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777037.660082:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18752 00000400:00000200:2.0:1775777037.660092:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777037.660098:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777037.660101:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777037.660104:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660108:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660110:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660112:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777037.660116:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777037.660118:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775777037.660120:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777037.660123:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775777037.660126:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775777037.660128:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775777037.660131:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660133:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660135:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660137:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777037.660139:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775777037.660143:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660145:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777037.660148:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777037.660151:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777037.660154:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660156:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775777037.660159:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777037.660170:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7f100. 00000020:00000040:2.0:1775777037.660175:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775777037.660178:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777037.660180:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660183:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777037.660186:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777037.660190:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777037.660198:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507800. 00000100:00000001:2.0:1775777037.660205:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777037.660214:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777037.660224:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777037.660229:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777037.660234:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048605922] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777037.660262:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777037.660266:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777037.660269:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775777037.660271:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777037.660272:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777037.660275:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777037.660277:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18752 00000400:00000200:2.0:1775777037.660281:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777037.660283:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777037.660285:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777037.660288:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660290:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660292:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660293:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777037.660295:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777037.660297:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775777037.660298:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777037.660301:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775777037.660302:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775777037.660304:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775777037.660307:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660309:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660311:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660312:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777037.660315:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775777037.660317:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660319:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777037.660321:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777037.660327:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777037.660329:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660331:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777037.660334:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777037.660337:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660339:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777037.660342:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d500. 00000020:00000040:2.0:1775777037.660345:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775777037.660348:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777037.660349:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660350:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777037.660352:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777037.660353:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777037.660356:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505400. 00000100:00000001:2.0:1775777037.660359:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777037.660364:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777037.660371:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777037.660372:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777037.660376:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000855db814] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777037.660388:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777037.660391:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777037.660394:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775777037.660395:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777037.660396:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777037.660399:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777037.660402:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18752 00000400:00000200:2.0:1775777037.660405:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777037.660408:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777037.660411:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777037.660413:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660415:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660417:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660419:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777037.660421:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777037.660422:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775777037.660423:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777037.660425:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775777037.660427:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775777037.660430:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775777037.660432:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660445:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660447:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660449:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777037.660451:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775777037.660453:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660455:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777037.660457:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777037.660460:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777037.660462:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660463:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777037.660465:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777037.660467:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660469:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777037.660472:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7c000. 00000020:00000040:2.0:1775777037.660474:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775777037.660477:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777037.660478:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660479:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777037.660481:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777037.660482:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777037.660486:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51504400. 00000100:00000001:2.0:1775777037.660489:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777037.660494:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777037.660501:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777037.660503:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777037.660506:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4798067] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777037.660513:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777037.660516:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777037.660518:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775777037.660520:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777037.660521:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777037.660523:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777037.660525:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18752 00000400:00000200:2.0:1775777037.660529:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777037.660531:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777037.660533:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777037.660536:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660537:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660539:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660541:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777037.660543:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777037.660544:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775777037.660545:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777037.660547:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775777037.660548:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775777037.660550:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775777037.660553:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660555:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660556:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660558:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777037.660560:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775777037.660562:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660564:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777037.660566:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777037.660568:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777037.660570:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660571:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777037.660574:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777037.660587:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660588:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777037.660591:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e300. 00000020:00000040:2.0:1775777037.660594:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775777037.660596:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777037.660597:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660599:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777037.660600:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777037.660602:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777037.660605:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507400. 00000100:00000001:2.0:1775777037.660608:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777037.660613:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777037.660620:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777037.660622:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777037.660626:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ff8c8494] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777037.660633:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777037.660635:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777037.660638:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775777037.660639:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777037.660641:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777037.660643:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777037.660645:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18752 00000400:00000200:2.0:1775777037.660649:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777037.660652:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777037.660654:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777037.660656:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660657:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660659:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660661:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777037.660663:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777037.660664:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775777037.660665:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777037.660667:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775777037.660668:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775777037.660670:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775777037.660673:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660675:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777037.660676:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660678:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777037.660680:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775777037.660682:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777037.660685:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777037.660686:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777037.660689:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777037.660690:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660692:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777037.660694:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777037.660696:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660698:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777037.660701:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7fb80. 00000020:00000040:2.0:1775777037.660703:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775777037.660705:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777037.660706:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777037.660708:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777037.660709:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777037.660710:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777037.660714:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505000. 00000100:00000001:2.0:1775777037.660717:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777037.660720:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777037.660726:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777037.660728:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777037.660730:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b1787ebf] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777037.660736:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777037.660740:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18763) 00000100:00000001:0.0:1775777037.661097:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777037.661101:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777037.661103:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777037.661112:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661119:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777037.661121:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.661125:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777037.661127:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777037.661130:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777037.661132:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099825792:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777037.661136:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777037.661140:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777037.661142:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.661145:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777037.661151:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777037.661153:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777037.661158:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777037.661161:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777037.661169:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777037.661175:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099825792, portal 25 00000100:00000001:0.0:1775777037.661177:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777037.661179:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288723200 : -114841420828416 : ffff978d65e7f100) 00000100:00000040:0.0:1775777037.661183:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661191:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777037.661193:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777037.661195:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777037.661198:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099825792, offset 0 00000400:00000200:0.0:1775777037.661204:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777037.661212:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.661218:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777037.661223:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777037.661226:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777037.661230:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777037.661233:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777037.661237:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4682:4682:256:4294967295] 192.168.201.139@tcp LPNI seq info [4682:4682:8:4294967295] 00000400:00000200:0.0:1775777037.661241:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.661252:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777037.661259:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.661263:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777037.661270:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777037.661274:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775777037.661280:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777037.661283:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775777037.661287:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.661290:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777037.661296:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777037.661301:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777037.661306:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777037.661308:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777037.661312:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777037.661316:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661324:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.661327:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661337:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.661339:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777037.661344:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777037.661347:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777037.661350:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313426678 00000100:00000010:0.0:1775777037.661354:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777037.661360:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777037.661366:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313426678 00000100:00000010:0.0:1775777037.661369:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237960. 00000100:00000001:0.0:1775777037.661372:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777037.661375:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777037.661377:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.661379:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777037.661383:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777037/real 1775777037] req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777037.661391:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777037.661393:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777037.661395:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777037.661397:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777037.661399:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777037.661401:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777037.661405:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777037.661412:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661418:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777037.661419:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.661422:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.661423:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777037.661427:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661444:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777037.661449:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661455:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777037.661458:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.661460:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.661461:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.661463:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777037.661465:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777037.661467:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777037.661471:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777037.661473:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777037.661478:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777037.661485:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099825792:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777037.661490:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777037.661494:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661500:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777037.661502:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.661504:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777037.661506:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777037.661508:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777037.661511:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825920:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777037.661513:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777037.661516:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777037.661517:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.661520:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777037.661523:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777037.661526:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777037.661529:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777037.661531:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777037.661535:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777037.661537:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099825920, portal 10 00000100:00000001:0.0:1775777037.661539:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777037.661540:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288716032 : -114841420835584 : ffff978d65e7d500) 00000100:00000040:0.0:1775777037.661547:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661553:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777037.661555:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777037.661557:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777037.661559:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099825920, offset 0 00000400:00000200:0.0:1775777037.661563:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777037.661566:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.661570:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777037.661572:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777037.661708:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777037.661711:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777037.661713:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777037.661715:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4683:4683:256:4294967295] 192.168.201.139@tcp LPNI seq info [4683:4683:8:4294967295] 00000400:00000200:0.0:1775777037.661720:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.661725:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777037.661730:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.661734:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777037.661739:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777037.661741:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775777037.661745:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777037.661747:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775777037.661751:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.661752:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777037.661756:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777037.661759:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777037.661762:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777037.661764:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777037.661766:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777037.661770:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661813:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.661817:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661824:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.661825:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777037.661828:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777037.661832:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777037.661835:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777037.661837:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313425414 00000100:00000001:0.0:1775777037.661840:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313426678 00000100:00000001:0.0:1775777037.661842:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313425414 00000100:00000001:0.0:1775777037.661844:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313426678 00000100:00000001:0.0:1775777037.661846:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777037.661848:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.661850:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777037.661853:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777037/real 1775777037] req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777037.661861:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777037.661862:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777037.661864:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777037.661866:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777037.661868:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777037.661870:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777037.661873:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777037.661881:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661887:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777037.661889:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.661890:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.661891:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777037.661895:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661901:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777037.661904:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661911:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777037.661914:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.661915:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.661916:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.661918:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777037.661920:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777037.661922:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777037.661925:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777037.661927:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777037.661930:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777037.661938:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099825920:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777037.661942:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777037.661945:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.661956:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777037.661957:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.661959:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777037.661961:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777037.661963:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777037.661965:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826048:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777037.661968:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777037.661970:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777037.661971:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.661973:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777037.661976:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777037.661978:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777037.661981:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777037.661984:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777037.661987:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777037.661990:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099826048, portal 10 00000100:00000001:0.0:1775777037.661992:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777037.661993:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288710656 : -114841420840960 : ffff978d65e7c000) 00000100:00000040:0.0:1775777037.661998:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.662004:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777037.662006:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777037.662007:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777037.662009:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099826048, offset 0 00000400:00000200:0.0:1775777037.662012:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777037.662016:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.662019:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777037.662021:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777037.662023:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777037.662025:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777037.662027:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777037.662031:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4684:4684:256:4294967295] 192.168.201.139@tcp LPNI seq info [4684:4684:8:4294967295] 00000400:00000200:0.0:1775777037.662034:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.662677:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777037.662693:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.662700:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777037.662707:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777037.662709:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775777037.662714:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777037.662717:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775777037.662721:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.662722:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777037.662727:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777037.662731:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777037.662735:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777037.662737:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777037.662740:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777037.662744:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.662754:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.662757:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.662763:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.662765:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777037.662770:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777037.662775:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777037.662812:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777037.662815:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313425416 00000100:00000001:0.0:1775777037.662817:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313426678 00000100:00000001:0.0:1775777037.662819:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313425416 00000100:00000001:0.0:1775777037.662821:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313426678 00000100:00000001:0.0:1775777037.662823:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777037.662826:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.662828:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777037.662831:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777037/real 1775777037] req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777037.662840:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777037.662841:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777037.662842:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777037.662844:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777037.662846:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777037.662848:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777037.662850:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777037.662856:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.662861:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777037.662863:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.662865:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.662867:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777037.662869:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.662875:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777037.662878:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.662885:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777037.662887:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.662888:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.662890:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.662892:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777037.662894:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777037.662896:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777037.662901:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777037.662903:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777037.662907:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777037.662914:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826048:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777037.662919:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777037.662922:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.662929:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777037.662930:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.662933:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777037.662935:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777037.662937:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777037.662940:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826176:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777037.662943:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777037.662946:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777037.662948:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.662950:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777037.662954:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775777037.662955:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777037.662959:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777037.662962:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777037.662965:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777037.662969:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099826176, portal 4 00000100:00000001:0.0:1775777037.662971:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777037.662972:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288719616 : -114841420832000 : ffff978d65e7e300) 00000100:00000040:0.0:1775777037.662977:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.662984:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777037.662987:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777037.662988:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777037.662991:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099826176, offset 0 00000400:00000200:0.0:1775777037.662995:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777037.663000:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.663004:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777037.663008:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777037.663010:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777037.663013:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777037.663016:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777037.663019:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4685:4685:256:4294967295] 192.168.201.139@tcp LPNI seq info [4685:4685:8:4294967295] 00000400:00000200:0.0:1775777037.663023:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.663029:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777037.663033:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.663036:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777037.663040:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777037.663465:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775777037.663476:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777037.663479:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775777037.663482:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.663484:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777037.663488:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777037.663491:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777037.663496:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777037.663498:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777037.663501:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777037.663506:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.663513:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.663516:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.663521:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.663523:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777037.663527:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777037.663531:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777037.663534:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777037.663536:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313425425 00000100:00000001:0.0:1775777037.663539:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313426678 00000100:00000001:0.0:1775777037.663541:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313425425 00000100:00000001:0.0:1775777037.663543:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313426678 00000100:00000001:0.0:1775777037.663545:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777037.663548:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.663549:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777037.663553:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777037/real 1775777037] req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777037.663561:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777037.663563:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777037.663565:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777037.663567:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777037.663569:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777037.663571:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777037.663573:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777037.664702:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.664710:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777037.664714:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.664717:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.664718:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777037.664722:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.664729:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777037.664732:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.664739:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777037.664742:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.664743:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.664744:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.664747:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777037.664749:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777037.664752:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777037.665258:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777037.665262:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777037.665267:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777037.665275:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826176:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777037.665281:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777037.665285:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.665292:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777037.665293:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.665297:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777037.665300:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777037.665303:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777037.665306:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826304:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777037.665309:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777037.665313:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777037.665316:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777037.665319:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777037.665327:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777037.665329:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777037.665335:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777037.665339:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777037.666220:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777037.666236:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099826304, portal 4 00000100:00000001:0.0:1775777037.666239:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777037.666241:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288725888 : -114841420825728 : ffff978d65e7fb80) 00000100:00000040:0.0:1775777037.666252:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.666260:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777037.666262:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777037.666265:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777037.666268:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099826304, offset 0 00000400:00000200:0.0:1775777037.666275:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777037.666284:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.666291:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777037.666296:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777037.666299:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777037.666303:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777037.666305:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777037.666309:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4686:4686:256:4294967295] 192.168.201.139@tcp LPNI seq info [4686:4686:8:4294967295] 00000400:00000200:0.0:1775777037.666313:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777037.666324:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777037.666330:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.666336:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777037.666342:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777037.666346:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775777037.666351:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777037.666354:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775777037.666358:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777037.666361:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:0.0:1775777037.666369:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777037.666374:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777037.666381:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:0.0:1775777037.666383:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:0.0:1775777037.666388:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777037.666392:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.666403:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.666406:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.666412:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.666414:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777037.666420:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777037.666424:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777037.666428:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777037.666431:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313425419 00000100:00000001:0.0:1775777037.666444:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313426678 00000100:00000001:0.0:1775777037.666447:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313425419 00000100:00000001:0.0:1775777037.666449:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313426678 00000100:00000001:0.0:1775777037.666452:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777037.666455:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.666457:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777037.666462:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777037/real 1775777037] req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777037.666473:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777037.666474:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777037.666476:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777037.666479:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777037.666481:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777037.666484:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777037.666487:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777037.666494:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.666505:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777037.666507:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.666509:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.666510:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777037.666514:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.666520:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777037.666524:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.666531:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777037.666533:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.666535:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777037.666536:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.666538:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777037.666541:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777037.666544:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777037.666548:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777037.666551:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777037.666555:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777037.666562:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826304:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777037.666566:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.666568:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.666573:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7f100 x1862033099825792/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.671632:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777037.671636:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777037.671644:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777037.671654:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777037.671657:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777037.671660:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777037.671662:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777037.671665:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507800. 02000000:00000001:0.0:1775777037.671669:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777037.671671:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777037.671673:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7f100. 00000100:00000001:0.0:1775777037.671677:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777037.671679:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.671681:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.671693:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d500 x1862033099825920/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.671700:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777037.671702:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777037.671703:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777037.671706:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777037.671707:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777037.671708:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777037.671710:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777037.671711:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505400. 02000000:00000001:0.0:1775777037.671714:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777037.671715:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777037.671716:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d500. 00000100:00000001:0.0:1775777037.671718:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777037.671719:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.671721:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.671724:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7c000 x1862033099826048/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.671731:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777037.671732:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777037.671734:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777037.671736:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777037.671737:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777037.671738:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777037.671740:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777037.671741:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51504400. 02000000:00000001:0.0:1775777037.671743:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777037.671744:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777037.671745:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7c000. 00000100:00000001:0.0:1775777037.671747:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777037.671748:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.671750:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.671754:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e300 x1862033099826176/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.671759:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777037.671760:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777037.671762:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775777037.671764:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777037.671765:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777037.671766:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777037.671767:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777037.671768:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507400. 02000000:00000001:0.0:1775777037.671770:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777037.671771:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777037.671772:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e300. 00000100:00000001:0.0:1775777037.671774:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777037.671775:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.671777:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777037.671798:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7fb80 x1862033099826304/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777037.671804:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777037.671805:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777037.671806:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777037.671810:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777037.671811:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777037.671812:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777037.671813:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777037.671814:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505000. 02000000:00000001:0.0:1775777037.671816:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777037.671817:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777037.671818:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7fb80. 00000100:00000001:0.0:1775777037.671820:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777037.671821:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.671823:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777037.671832:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777037.671833:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777037.671836:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777037.671838:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777037.852185:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777037.852191:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777037.852193:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777037.852199:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777037.852203:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777037.852207:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777037.852209:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777037.852212:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777037.852214:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777038.364087:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777038.364095:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777038.364097:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777038.364102:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777038.364106:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777038.364110:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777038.364113:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777038.364115:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777038.364118:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777038.428015:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777038.428021:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777038.428028:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777038.428094:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777038.428099:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777038.428100:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777038.428104:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777038.428115:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777038.428731:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777038.428735:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777038.428739:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777038.877027:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777038.877037:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777038.877039:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777038.877046:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777038.877051:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777038.877055:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777038.877058:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777038.877061:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777038.877063:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777038.939934:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313426678, at 4313426688 00000100:00000010:1.0:1775777038.939948:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237960. 00000100:00000040:0.0:1775777038.940202:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313426678, at 4313426688 00000100:00000010:0.0:1775777038.940209:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775777039.388332:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777039.388340:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777039.388342:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777039.388348:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777039.388352:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777039.388356:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777039.388358:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777039.388360:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777039.388362:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777039.451963:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777039.451971:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777039.451974:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777039.451977:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777039.451982:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777039.451984:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777039.451986:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777039.451988:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777039.451992:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777039.451994:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777039.451997:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777039.451998:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777039.452001:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777039.452002:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777039.452109:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777039.452112:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777039.452115:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777039.899931:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777039.899938:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777039.899941:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777039.899946:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777039.899950:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777039.899954:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777039.899957:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777039.899960:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777039.899963:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777040.411931:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777040.411939:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777040.411940:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777040.411946:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777040.411950:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777040.411955:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777040.411958:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777040.411960:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777040.411962:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777040.476063:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777040.476069:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777040.476075:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777040.476077:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777040.476079:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777040.476080:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777040.476084:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777040.476085:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777040.476187:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777040.476191:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777040.476194:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777040.926020:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777040.926027:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777040.926029:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777040.926046:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777040.926051:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777040.926055:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777040.926057:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777040.926059:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777040.926062:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777041.435914:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777041.435923:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777041.435925:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777041.435931:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777041.435935:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777041.435939:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777041.435942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777041.435944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777041.435947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777041.499918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777041.499924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777041.499930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777041.499932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777041.499935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777041.499936:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777041.499940:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777041.499941:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777041.500130:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777041.500134:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777041.500137:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777041.950928:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777041.950935:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777041.950936:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777041.950941:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777041.950945:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777041.950949:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777041.950951:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777041.950954:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777041.950956:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.460014:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777042.460024:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777042.460026:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777042.460032:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777042.460037:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777042.460041:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777042.460044:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777042.460046:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777042.460049:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.525971:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777042.525977:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.525982:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777042.525986:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777042.525988:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777042.525993:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.525995:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777042.525998:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526000:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526002:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526005:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777042.526006:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777042.526007:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777042.526009:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526010:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777042.526011:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526013:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526014:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526017:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777042.526018:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777042.526019:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777042.526020:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526021:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777042.526023:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526024:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777042.526026:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777042.526121:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777042.526124:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777042.526127:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775777042.785378:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777042.785398:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775777042.785402:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777042.785413:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777042.785419:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777042.785427:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18758 00000400:00000200:3.0:1775777042.785437:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777042.785455:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777042.785459:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777042.785462:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785465:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.785468:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785470:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777042.785474:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777042.785476:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775777042.785479:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777042.785482:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775777042.785484:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775777042.785487:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775777042.785489:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785491:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.785493:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785495:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777042.785498:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775777042.785502:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785504:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777042.785507:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777042.785510:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777042.785513:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.785515:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775777042.785518:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777042.785529:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedf80. 00000020:00000040:3.0:1775777042.785532:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775777042.785536:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777042.785538:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.785541:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777042.785544:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777042.785549:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777042.785558:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775777042.785564:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777042.785573:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777042.785583:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777042.785587:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777042.785591:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29f8335] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777042.785618:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777042.785621:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777042.785624:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775777042.785626:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777042.785627:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777042.785630:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777042.785632:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18758 00000400:00000200:3.0:1775777042.785636:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777042.785640:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777042.785642:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777042.785644:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785646:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.785648:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785650:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777042.785652:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777042.785653:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775777042.785655:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777042.785657:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775777042.785659:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775777042.785661:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775777042.785664:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785665:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.785667:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785669:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777042.785671:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775777042.785673:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785676:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777042.785677:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777042.785682:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777042.785684:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.785686:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777042.785690:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777042.785693:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.785695:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777042.785700:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def800. 00000020:00000040:3.0:1775777042.785702:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775777042.785705:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777042.785706:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.785708:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777042.785709:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777042.785711:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777042.785715:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775777042.785718:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777042.785722:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777042.785730:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777042.785732:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777042.785735:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000507067a7] to pc [ptlrpcd_rcv+-1] req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777042.785743:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777042.785745:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777042.785748:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775777042.785750:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777042.785751:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777042.785754:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777042.785756:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18758 00000400:00000200:3.0:1775777042.785760:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777042.785762:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777042.785765:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777042.785768:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785770:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.785771:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.785773:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777042.785775:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777042.785776:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775777042.785805:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777042.785806:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000100:00000001:0.0:1775777042.786202:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777042.786208:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777042.786211:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777042.786218:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.786227:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777042.786229:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.786232:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777042.786234:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777042.786237:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777042.786240:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099826432:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777042.786244:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777042.786248:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777042.786250:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.786253:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777042.786258:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777042.786260:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777042.786265:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777042.786270:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513066e0. 00000400:00000010:0.0:1775777042.786277:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777042.786285:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099826432, portal 25 00000100:00000001:0.0:1775777042.786287:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777042.786288:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257984 : -114841958293632 : ffff978d45dedf80) 00000100:00000040:0.0:1775777042.786294:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.786301:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777042.786304:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777042.786306:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8330. 00000100:00000200:0.0:1775777042.786309:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099826432, offset 0 00000400:00000200:0.0:1775777042.786316:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777042.786335:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.786341:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777042.786356:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777042.786361:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777042.786364:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777042.786367:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777042.786371:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4687:4687:256:4294967295] 192.168.201.139@tcp LPNI seq info [4687:4687:8:4294967295] 00000400:00000200:0.0:1775777042.786375:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.786396:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777042.786403:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777042.786408:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bc00. 00000800:00000200:0.0:1775777042.786415:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777042.786420:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775777042.786450:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777042.786454:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777042.786457:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313431803 00000100:00000010:0.0:1775777042.786462:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777042.786468:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777042.786476:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313431803 00000100:00000010:0.0:1775777042.786479:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237480. 00000100:00000001:0.0:1775777042.786483:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777042.786486:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777042.786488:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.786491:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777042.786495:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.786502:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777042.786503:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.786505:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777042.786507:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777042.786509:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777042.786511:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826560:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777042.786515:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777042.786516:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777042.786518:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.786519:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777042.786522:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775777042.786524:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777042.786527:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777042.786530:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777042.786534:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777042.786537:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099826560, portal 10 00000100:00000001:0.0:1775777042.786538:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777042.786540:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751264256 : -114841958287360 : ffff978d45def800) 00000100:00000040:0.0:1775777042.786546:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.786553:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777042.786555:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777042.786558:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8b28. 00000100:00000200:0.0:1775777042.786560:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099826560, offset 0 00000400:00000200:0.0:1775777042.786564:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777042.786568:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.786573:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777042.786575:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777042.786578:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777042.786580:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777042.786582:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777042.786585:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4688:4688:255:4294967295] 192.168.201.139@tcp LPNI seq info [4688:4688:7:4294967295] 00000400:00000200:0.0:1775777042.786589:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.786595:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777042.786599:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777042.786603:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b300. 00000800:00000200:0.0:1775777042.786606:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777042.786610:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775777042.786612:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777042.786616:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777042.786619:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777042.786621:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313430538 00000100:00000001:0.0:1775777042.786624:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313431803 00000100:00000001:0.0:1775777042.786626:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313430538 00000100:00000001:0.0:1775777042.786628:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313431803 00000100:00000001:0.0:1775777042.786630:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777042.786632:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.786635:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.786637:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777042.786645:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777042.786647:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775777042.786649:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777042.786651:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777042.786653:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.786654:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775777042.787138:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775777042.787142:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775777042.787147:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787149:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.787152:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787154:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777042.787156:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775777042.787159:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787162:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777042.787164:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777042.787168:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777042.787170:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787172:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777042.787175:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777042.787178:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787179:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777042.787184:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deea00. 00000020:00000040:3.0:1775777042.787187:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775777042.787189:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777042.787191:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787193:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777042.787196:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777042.787199:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777042.787203:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775777042.787208:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777042.787213:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777042.787222:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777042.787226:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777042.787229:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f7476cc] to pc [ptlrpcd_rcv+-1] req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777042.787252:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777042.787256:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777042.787260:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775777042.787261:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777042.787263:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777042.787266:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777042.787268:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18758 00000400:00000200:3.0:1775777042.787272:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777042.787276:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777042.787278:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777042.787280:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787282:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.787284:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787286:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777042.787288:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777042.787289:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775777042.787291:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777042.787292:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775777042.787294:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775777042.787296:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775777042.787298:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787300:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.787302:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787304:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777042.787306:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775777042.787309:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787311:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777042.787312:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777042.787315:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777042.787317:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787319:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777042.787321:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777042.787323:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787325:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777042.787328:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded180. 00000020:00000040:3.0:1775777042.787329:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775777042.787332:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777042.787333:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787334:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777042.787335:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777042.787337:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777042.787340:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775777042.787342:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777042.787359:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777042.787365:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777042.787367:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777042.787370:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000096b6df22] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777042.787376:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777042.787378:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777042.787380:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775777042.787382:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777042.787646:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777042.787649:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777042.787652:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18758 00000400:00000200:3.0:1775777042.787656:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777042.787659:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777042.787660:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777042.787662:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787664:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.787666:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787668:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777042.787670:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777042.787672:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775777042.787673:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777042.787675:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775777042.787676:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775777042.787678:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775777042.787680:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787681:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777042.787683:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787685:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777042.787687:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775777042.787690:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777042.787692:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777042.787693:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777042.787696:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777042.787698:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787699:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777042.787701:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777042.787703:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787705:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777042.787709:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee300. 00000020:00000040:3.0:1775777042.787711:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775777042.787713:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777042.787714:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.787716:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777042.787717:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777042.787719:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777042.787723:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775777042.787727:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777042.787731:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777042.787743:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777042.787745:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777042.787749:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015048b5d] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777042.787756:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777042.787761:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18768) 00000100:00000001:0.0:1775777042.788321:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777042.788326:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777042.788328:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777042.788334:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.788342:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777042.788357:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.788360:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777042.788362:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777042.788364:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777042.788367:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826688:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777042.788372:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777042.788375:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777042.788377:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.788379:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777042.788384:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777042.788386:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777042.788400:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777042.788404:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306318. 00000400:00000010:0.0:1775777042.788411:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8220. 00000100:00000200:0.0:1775777042.788417:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099826688, portal 10 00000100:00000001:0.0:1775777042.788419:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777042.788421:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751260672 : -114841958290944 : ffff978d45deea00) 00000100:00000040:0.0:1775777042.788427:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.788434:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777042.788437:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777042.788450:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8660. 00000100:00000200:0.0:1775777042.788454:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099826688, offset 0 00000400:00000200:0.0:1775777042.788461:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777042.788467:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.788473:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777042.788477:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777042.788480:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777042.788483:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777042.788486:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777042.788490:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4689:4689:254:4294967295] 192.168.201.139@tcp LPNI seq info [4689:4689:6:4294967295] 00000400:00000200:0.0:1775777042.788494:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.788503:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777042.788510:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777042.788514:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b000. 00000800:00000200:0.0:1775777042.788522:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777042.788527:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775777042.788531:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777042.788535:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777042.788538:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777042.788541:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313430541 00000100:00000001:0.0:1775777042.788544:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313431803 00000100:00000001:0.0:1775777042.788546:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313430541 00000100:00000001:0.0:1775777042.788549:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313431803 00000100:00000001:0.0:1775777042.788551:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777042.788553:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.788556:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777042.788560:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.788566:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777042.788568:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.788570:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777042.788572:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777042.788573:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777042.788576:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826816:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777042.788579:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777042.788581:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777042.788583:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.788585:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777042.788588:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777042.788590:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777042.788594:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777042.788596:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306dc0. 00000400:00000010:0.0:1775777042.788600:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8198. 00000100:00000200:0.0:1775777042.788603:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099826816, portal 4 00000100:00000001:0.0:1775777042.788605:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777042.788606:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751254400 : -114841958297216 : ffff978d45ded180) 00000100:00000040:0.0:1775777042.788612:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.788617:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777042.788619:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777042.788622:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b87f8. 00000100:00000200:0.0:1775777042.788624:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099826816, offset 0 00000400:00000200:0.0:1775777042.788629:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777042.788632:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.788635:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777042.788637:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777042.788639:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777042.788641:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777042.788644:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777042.788646:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4690:4690:253:4294967295] 192.168.201.139@tcp LPNI seq info [4690:4690:5:4294967295] 00000400:00000200:0.0:1775777042.788650:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.788656:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777042.788660:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777042.788664:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9bd00. 00000800:00000200:0.0:1775777042.788667:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777042.788671:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775777042.788673:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777042.788676:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777042.788679:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777042.788681:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313430550 00000100:00000001:0.0:1775777042.788683:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313431803 00000100:00000001:0.0:1775777042.788685:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313430550 00000100:00000001:0.0:1775777042.788687:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313431803 00000100:00000001:0.0:1775777042.788689:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777042.788691:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.788693:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777042.788697:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.788703:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777042.788704:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.788706:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777042.788707:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777042.788710:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777042.788712:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826944:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777042.788715:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777042.788716:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777042.788718:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777042.788719:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777042.788722:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777042.788724:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777042.788727:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777042.788729:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d51306b58. 00000400:00000010:0.0:1775777042.788732:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8bb0. 00000100:00000200:0.0:1775777042.788735:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099826944, portal 4 00000100:00000001:0.0:1775777042.788737:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777042.788738:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751258880 : -114841958292736 : ffff978d45dee300) 00000100:00000040:0.0:1775777042.788744:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777042.788750:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777042.788752:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777042.788754:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8e58. 00000100:00000200:0.0:1775777042.788756:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099826944, offset 0 00000400:00000200:0.0:1775777042.788760:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777042.788764:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.788768:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777042.788771:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777042.788773:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777042.788775:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777042.788777:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777042.789604:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4691:4691:252:4294967295] 192.168.201.139@tcp LPNI seq info [4691:4691:4:4294967295] 00000400:00000200:0.0:1775777042.789610:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777042.789618:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777042.789624:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777042.789631:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9b200. 00000800:00000200:0.0:1775777042.789636:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777042.789640:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:0.0:1775777042.789644:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777042.789648:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777042.789651:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777042.789652:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313430542 00000100:00000001:0.0:1775777042.789654:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313431803 00000100:00000001:0.0:1775777042.789657:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313430542 00000100:00000001:0.0:1775777042.789658:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313431803 00000100:00000001:0.0:1775777042.789660:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777042.789663:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.789667:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.789670:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777042.789677:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777042.789678:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775777042.789680:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777042.789682:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777042.789684:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777042.789685:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777042.972092:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777042.972100:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777042.972102:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777042.972108:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777042.973710:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777042.973714:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777042.973717:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777042.973719:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777042.973722:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777043.484024:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777043.484032:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777043.484035:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777043.484042:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777043.484046:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777043.484052:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777043.484054:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777043.484057:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777043.484060:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777043.547917:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777043.547924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777043.547930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777043.547932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777043.547935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777043.551496:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777043.551520:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777043.551522:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775777043.551534:0:314648:0:(ldlm_pool.c:1331:ldlm_pools_recalc_task()) Negative interval(0) 00000020:00000001:0.0:1775777043.551648:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777043.551651:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777043.551655:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777043.995981:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777043.995987:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777043.995988:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777043.995994:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777043.995997:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777043.996001:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777043.996004:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777043.996006:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777043.996009:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777044.060019:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313431803, at 4313431808 00000100:00000010:0.0:1775777044.060035:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775777044.060508:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313431803, at 4313431808 00000100:00000010:1.0:1775777044.060516:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237480. 00000008:00000020:2.0:1775777044.316814:0:314648:0:(osc_request.c:745:osc_update_next_shrink()) next time 19964 to shrink grant 00000008:00000020:2.0:1775777044.316824:0:314648:0:(osc_request.c:745:osc_update_next_shrink()) next time 19964 to shrink grant 00000100:00000001:3.0:1775777044.508004:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777044.508013:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777044.508015:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777044.508020:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777044.508025:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777044.508029:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777044.508031:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777044.508034:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777044.508037:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777044.572481:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777044.572496:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777044.572502:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777044.572504:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777044.572508:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777044.572509:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777044.572512:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777044.572513:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777044.572910:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777044.572914:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777044.572918:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777045.025933:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777045.025941:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777045.025943:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777045.025949:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777045.025953:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777045.025956:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777045.025959:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777045.025961:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777045.025964:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777045.532005:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777045.532012:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777045.532014:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777045.532020:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777045.532024:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777045.532029:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777045.532032:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777045.532048:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777045.532051:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777045.599015:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777045.599022:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777045.599028:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777045.599029:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777045.599043:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777045.599045:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777045.599049:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777045.599050:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777045.602260:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777045.602268:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777045.602272:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775777045.857528:0:301878:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.139:988 00000400:00000100:1.0:1775777045.857573:0:301878:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.139@tcp at host 192.168.201.139 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1775777045.857584:0:301878:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777045.857588:0:301878:0:(router.c:1750:lnet_notify()) 192.168.201.39@tcp notifying 192.168.201.139@tcp: down 00000400:00000200:1.0:1775777045.857593:0:301878:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777045.857597:0:301878:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:1.0:1775777045.857603:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775777045.857606:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bc00. 00000400:00000200:1.0:1775777045.857613:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775777045.857617:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775777045.857623:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8330 00000400:00000010:1.0:1775777045.857625:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8330. 00000100:00000001:1.0:1775777045.857630:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775777045.857638:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857665:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775777045.857670:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857677:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777045.857678:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775777045.857687:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775777045.857689:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b300. 00000400:00000200:1.0:1775777045.857692:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775777045.857694:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775777045.857697:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8b28 00000400:00000010:1.0:1775777045.857699:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8b28. 00000100:00000001:1.0:1775777045.857701:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775777045.857705:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857712:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775777045.857715:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857721:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777045.857722:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775777045.857727:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775777045.857729:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b000. 00000400:00000200:1.0:1775777045.857732:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775777045.857734:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775777045.857737:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8660 00000400:00000010:1.0:1775777045.857738:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8660. 00000100:00000001:1.0:1775777045.857740:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775777045.857744:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857751:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775777045.857753:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857758:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777045.857760:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775777045.857764:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775777045.857766:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9bd00. 00000400:00000200:1.0:1775777045.857768:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775777045.857771:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775777045.857773:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b87f8 00000400:00000010:1.0:1775777045.857774:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b87f8. 00000100:00000001:1.0:1775777045.857776:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775777045.857838:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857849:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775777045.857860:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857866:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777045.857867:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1775777045.857872:0:301878:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.139@tcp 00000800:00000010:1.0:1775777045.857875:0:301878:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9b200. 00000400:00000200:1.0:1775777045.857877:0:301878:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1775777045.857879:0:301878:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1775777045.857882:0:301878:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8e58 00000400:00000010:1.0:1775777045.857883:0:301878:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8e58. 00000100:00000001:1.0:1775777045.857885:0:301878:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775777045.857888:0:301878:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857894:0:301878:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775777045.857896:0:301878:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777045.857901:0:301878:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777045.857902:0:301878:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777045.861390:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777045.861410:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777045.861413:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777045.861437:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777042/real 1775777045] req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777045.861456:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777045.861458:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777045.861462:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306318. 00000400:00000200:0.0:1775777045.861468:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8220 00000400:00000010:0.0:1775777045.861470:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8220. 00000100:00000001:0.0:1775777045.861474:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777045.861477:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777045.861486:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861493:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777045.861495:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861498:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861500:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777045.861503:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861510:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777045.861514:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861521:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777045.861524:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861526:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861527:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861530:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777045.861532:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777045.861536:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777045.861541:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777045.861544:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777045.861548:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777045.861555:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826688:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777045.861560:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777045.861564:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777042/real 1775777045] req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777045.861571:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777045.861572:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777045.861574:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306dc0. 00000400:00000200:0.0:1775777045.861576:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8198 00000400:00000010:0.0:1775777045.861578:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8198. 00000100:00000001:0.0:1775777045.861580:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777045.861582:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777045.861589:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861595:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777045.861597:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861598:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861600:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777045.861603:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861615:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777045.861618:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861623:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777045.861626:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861627:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861628:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861629:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777045.861631:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777045.861633:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777045.861637:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777045.861639:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777045.861643:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777045.861650:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826816:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777045.861654:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777045.861658:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777042/real 1775777045] req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777045.861666:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777045.861667:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777045.861669:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d51306b58. 00000400:00000200:0.0:1775777045.861671:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8bb0 00000400:00000010:0.0:1775777045.861672:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8bb0. 00000100:00000001:0.0:1775777045.861674:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777045.861677:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777045.861703:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861708:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777045.861710:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861711:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861713:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777045.861716:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861730:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777045.861733:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861739:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777045.861741:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861742:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861743:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861745:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777045.861747:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777045.861749:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777045.861752:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777045.861754:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777045.861765:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777045.861772:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826944:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777045.861776:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777045.861824:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777042/real 1775777045] req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777045.861837:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777045.861838:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777045.861840:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513066e0. 00000400:00000200:0.0:1775777045.861843:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777045.861845:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777045.861847:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777045.861849:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777045.861856:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861863:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777045.861864:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861866:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861867:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777045.861870:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861877:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777045.861880:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861887:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777045.861889:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861890:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861891:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861893:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777045.861894:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777045.861897:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777045.861900:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777045.861902:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777045.861905:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777045.861912:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099826432:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777045.861916:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777045.861920:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777042/real 1775777045] req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777045.861926:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777045.861927:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777045.861930:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777045.861932:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777045.861934:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777045.861935:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777045.861938:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777045.861945:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861950:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777045.861951:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861953:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861954:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777045.861957:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861962:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777045.861965:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.861970:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777045.861973:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861974:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777045.861975:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.861977:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777045.861979:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777045.861981:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777045.861983:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777045.861985:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777045.861988:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777045.861995:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099826560:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777045.861999:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.862002:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777045.862005:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deea00 x1862033099826688/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.862013:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777045.862016:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777045.862020:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777045.862024:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777045.862026:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777045.862029:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777045.862031:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777045.862046:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:0.0:1775777045.862051:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777045.862053:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777045.862055:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deea00. 00000100:00000001:0.0:1775777045.862059:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777045.862060:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.862061:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777045.862065:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded180 x1862033099826816/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.862071:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777045.862072:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777045.862073:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777045.862076:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777045.862077:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777045.862079:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777045.862080:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777045.862082:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:0.0:1775777045.862084:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777045.862085:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777045.862086:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded180. 00000100:00000001:0.0:1775777045.862088:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777045.862090:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.862091:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777045.862094:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee300 x1862033099826944/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.862100:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777045.862101:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777045.862102:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777045.862104:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777045.862105:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777045.862107:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777045.862109:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777045.862110:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:0.0:1775777045.862112:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777045.862113:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777045.862114:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee300. 00000100:00000001:0.0:1775777045.862116:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777045.862117:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.862118:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777045.862121:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedf80 x1862033099826432/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.862127:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777045.862128:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777045.862129:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777045.862131:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777045.862132:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777045.862133:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777045.862135:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777045.862137:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:0.0:1775777045.862139:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777045.862140:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777045.862141:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedf80. 00000100:00000001:0.0:1775777045.862142:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777045.862144:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.862145:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777045.862147:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def800 x1862033099826560/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777045.862152:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777045.862153:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777045.862154:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775777045.862156:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777045.862157:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777045.862158:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777045.862160:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777045.862161:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:0.0:1775777045.862162:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777045.862163:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777045.862164:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def800. 00000100:00000001:0.0:1775777045.862166:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777045.862167:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.862169:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777045.862177:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777045.862179:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777045.862181:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777045.862182:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777046.044144:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777046.044151:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777046.044153:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777046.044157:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777046.044161:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777046.044164:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777046.044167:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777046.044169:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777046.044172:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777046.556740:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777046.556750:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777046.556752:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777046.556758:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777046.556763:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777046.556767:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777046.556770:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777046.556773:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777046.556794:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777046.620371:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777046.620378:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777046.620384:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777046.620385:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777046.620389:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777046.620391:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777046.620394:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777046.620395:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777046.620499:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777046.620503:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777046.620506:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777047.070384:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777047.070392:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777047.070394:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777047.070401:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777047.070406:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777047.070410:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777047.070413:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777047.070415:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777047.070418:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777047.579911:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777047.579919:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777047.579921:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777047.579927:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777047.579931:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777047.579936:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777047.579951:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777047.579954:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777047.579965:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777047.644990:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777047.644995:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777047.645001:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777047.645002:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777047.645006:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777047.645007:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777047.645010:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777047.645011:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777047.652980:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777047.652993:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777047.652999:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775777047.900233:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777047.900253:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775777047.900305:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777047.900310:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777047.900314:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777047.900323:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18763 00000400:00000200:1.0:1775777047.900332:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777047.900338:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777047.900341:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777047.900344:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900348:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900350:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900353:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777047.900356:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777047.900358:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775777047.900366:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777047.900369:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775777047.900371:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775777047.900374:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775777047.900376:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900378:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900380:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900382:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777047.900385:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775777047.900388:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900390:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777047.900393:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777047.900404:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777047.900407:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900409:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775777047.900412:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777047.900418:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6fb80. 00000020:00000040:1.0:1775777047.900422:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775777047.900426:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777047.900427:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900430:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777047.900432:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777047.900436:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777047.900443:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38400. 00000100:00000001:1.0:1775777047.900450:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777047.900459:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777047.900475:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777047.900480:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777047.900485:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000314d8ea9] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777047.900510:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777047.900514:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777047.900517:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775777047.900518:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777047.900520:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777047.900522:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777047.900524:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18763 00000400:00000200:1.0:1775777047.900528:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777047.900531:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777047.900534:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777047.900537:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900538:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900540:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900542:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777047.900544:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777047.900545:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775777047.900546:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777047.900548:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775777047.900550:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775777047.900551:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775777047.900554:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900556:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900557:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900559:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777047.900562:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775777047.900564:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900567:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777047.900569:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777047.900574:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777047.900576:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900579:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777047.900596:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775777047.900600:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900602:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777047.900606:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6dc00. 00000020:00000040:1.0:1775777047.900608:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775777047.900611:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777047.900612:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900613:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777047.900615:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777047.900616:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777047.900620:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39800. 00000100:00000001:1.0:1775777047.900624:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777047.900629:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777047.900636:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777047.900638:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777047.900641:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000807e1a80] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777047.900648:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777047.900651:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777047.900653:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775777047.900655:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777047.900656:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777047.900658:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777047.900661:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18763 00000400:00000200:1.0:1775777047.900665:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777047.900668:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777047.900670:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777047.900672:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900674:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900675:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900677:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777047.900679:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777047.900680:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775777047.900681:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777047.900683:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775777047.900685:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775777047.900686:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775777047.900689:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900691:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900692:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900695:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777047.900697:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775777047.900700:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900702:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777047.900703:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777047.900706:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777047.900708:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900710:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777047.900712:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775777047.900714:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900716:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777047.900719:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c380. 00000020:00000040:1.0:1775777047.900722:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775777047.900724:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777047.900725:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900727:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777047.900728:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777047.900730:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777047.900733:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39400. 00000100:00000001:1.0:1775777047.900736:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777047.900740:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777047.900746:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777047.900748:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777047.900751:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0c55087] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777047.900759:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777047.900761:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777047.900763:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775777047.900765:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777047.900766:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777047.900768:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777047.900771:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18763 00000400:00000200:1.0:1775777047.900774:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777047.900811:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777047.900814:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777047.900817:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900820:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900821:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900823:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777047.900825:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777047.900826:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775777047.900827:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777047.900829:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775777047.900831:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775777047.900833:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775777047.900836:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900837:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900839:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900841:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777047.900843:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775777047.900845:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900847:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777047.900849:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777047.900852:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777047.900854:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900855:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777047.900858:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775777047.900860:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900862:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777047.900865:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c700. 00000020:00000040:1.0:1775777047.900867:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775777047.900869:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777047.900870:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900872:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777047.900873:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777047.900874:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777047.900878:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a400. 00000100:00000001:1.0:1775777047.900881:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777047.900886:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777047.900892:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777047.900894:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777047.900897:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008ddf3b7] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777047.900904:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777047.900907:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777047.900910:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775777047.900912:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777047.900913:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777047.900915:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777047.900918:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18763 00000400:00000200:1.0:1775777047.900922:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777047.900924:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777047.900926:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777047.900929:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900931:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900933:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900935:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777047.900937:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777047.900938:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775777047.900939:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777047.900942:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775777047.900943:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775777047.900945:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775777047.900948:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900950:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777047.900952:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900954:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777047.900956:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775777047.900958:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777047.900961:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777047.900962:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777047.900965:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777047.900967:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900969:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777047.900971:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775777047.900973:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900975:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777047.900979:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f100. 00000020:00000040:1.0:1775777047.900981:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775777047.900984:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777047.900985:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777047.900986:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777047.900988:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777047.900989:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777047.900992:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b800. 00000100:00000001:1.0:1775777047.900995:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777047.901000:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777047.901006:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777047.901008:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777047.901011:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000070fd75f1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777047.901019:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777047.901023:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18773) 00000100:00000001:0.0:1775777047.902876:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777047.902887:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777047.902891:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777047.902905:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.902920:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777047.902922:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.902929:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777047.902933:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777047.902937:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777047.902941:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099827072:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777047.902947:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777047.902952:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777047.902956:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.902959:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777047.902974:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775777047.902976:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777047.902987:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777047.902993:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777047.903004:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777047.903016:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099827072, portal 25 00000100:00000001:0.0:1775777047.903019:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777047.903021:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287611776 : -114841421939840 : ffff978d65d6fb80) 00000100:00000040:0.0:1775777047.903027:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903036:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777047.903039:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777047.903042:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777047.903046:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099827072, offset 0 00000400:00000200:0.0:1775777047.903055:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777047.903066:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.903075:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777047.903080:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777047.903085:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777047.903091:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777047.903094:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777047.903099:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4692:4692:256:4294967295] 192.168.201.139@tcp LPNI seq info [4692:4692:8:4294967295] 00000400:00000200:0.0:1775777047.903104:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.903117:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777047.903125:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.903131:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777047.903139:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777047.903142:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775777047.903148:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777047.903151:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775777047.903172:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.903175:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777047.903183:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777047.903187:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777047.903192:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777047.903195:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777047.903200:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777047.903204:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903215:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.903218:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903225:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.903227:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777047.903232:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777047.903236:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777047.903239:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313436920 00000100:00000010:0.0:1775777047.903244:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777047.903252:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777047.903277:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313436920 00000100:00000010:0.0:1775777047.903281:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237b40. 00000100:00000001:0.0:1775777047.903284:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777047.903288:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777047.903291:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.903293:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777047.903298:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777047/real 1775777047] req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777047.903320:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777047.903322:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777047.903325:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777047.903327:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777047.903329:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777047.903331:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777047.903334:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777047.903342:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903348:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777047.903349:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.903351:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.903353:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777047.903357:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903364:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777047.903368:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903374:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777047.903377:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.903379:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.903380:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.903382:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777047.903384:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777047.903387:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777047.903391:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777047.903394:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777047.903551:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777047.903560:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099827072:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777047.903565:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777047.903569:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903575:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777047.903577:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.903579:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777047.903581:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777047.903583:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777047.903598:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827200:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777047.903601:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777047.903604:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777047.903605:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.903607:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777047.903611:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777047.903613:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777047.903618:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777047.903620:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777047.903623:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777047.903627:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099827200, portal 10 00000100:00000001:0.0:1775777047.903629:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777047.903630:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287603712 : -114841421947904 : ffff978d65d6dc00) 00000100:00000040:0.0:1775777047.903636:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903642:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777047.903644:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777047.903646:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777047.903648:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099827200, offset 0 00000400:00000200:0.0:1775777047.903652:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777047.903656:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.903660:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777047.903662:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777047.903664:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777047.903667:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777047.903669:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777047.903672:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4693:4693:256:4294967295] 192.168.201.139@tcp LPNI seq info [4693:4693:8:4294967295] 00000400:00000200:0.0:1775777047.903675:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.903682:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777047.903686:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.903689:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777047.903692:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777047.903695:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775777047.903698:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777047.903701:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775777047.903704:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.903706:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777047.903709:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777047.903711:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777047.903714:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777047.903716:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777047.903718:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777047.903722:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903728:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.903731:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903736:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.903738:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777047.903740:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777047.903744:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777047.903747:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777047.903748:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313435655 00000100:00000001:0.0:1775777047.903751:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313436920 00000100:00000001:0.0:1775777047.903753:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313435655 00000100:00000001:0.0:1775777047.903754:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313436920 00000100:00000001:0.0:1775777047.903756:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777047.903758:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.903760:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777047.903763:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777047/real 1775777047] req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777047.903769:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777047.903770:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777047.903772:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777047.903774:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777047.903775:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777047.903777:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777047.903841:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777047.903849:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903855:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777047.903857:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.903858:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.903859:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777047.903863:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903870:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777047.903873:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903880:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777047.903882:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.903884:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.903885:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.903887:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777047.903889:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777047.903891:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777047.903894:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777047.903896:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777047.903900:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777047.903907:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827200:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777047.903910:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777047.903913:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903919:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777047.903921:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.903922:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777047.903924:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777047.903926:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777047.903929:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827328:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777047.903932:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777047.903934:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777047.903935:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.903937:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777047.903940:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777047.903942:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777047.903945:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777047.903947:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777047.903950:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777047.903953:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099827328, portal 10 00000100:00000001:0.0:1775777047.903955:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777047.903956:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287597440 : -114841421954176 : ffff978d65d6c380) 00000100:00000040:0.0:1775777047.903961:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.903966:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777047.903969:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777047.903971:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777047.903973:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099827328, offset 0 00000400:00000200:0.0:1775777047.903977:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777047.903980:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.903983:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777047.903986:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777047.903988:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777047.903990:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777047.903992:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777047.903994:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4694:4694:256:4294967295] 192.168.201.139@tcp LPNI seq info [4694:4694:8:4294967295] 00000400:00000200:0.0:1775777047.903998:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.904003:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777047.904007:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.904010:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777047.904013:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777047.904016:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775777047.904019:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777047.904021:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775777047.904024:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.904025:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777047.904028:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777047.904031:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777047.904034:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777047.904036:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777047.904038:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777047.904041:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904048:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.904050:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904055:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.904057:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777047.904059:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777047.904061:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777047.904064:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777047.904065:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313435657 00000100:00000001:0.0:1775777047.904067:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313436920 00000100:00000001:0.0:1775777047.904069:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313435657 00000100:00000001:0.0:1775777047.904071:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313436920 00000100:00000001:0.0:1775777047.904072:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777047.904074:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.904076:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777047.904078:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777047/real 1775777047] req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777047.904085:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777047.904086:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777047.904087:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777047.904089:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777047.904091:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777047.904093:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777047.904095:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777047.904101:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904108:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777047.904109:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904111:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.904112:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777047.904115:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904121:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777047.904124:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904130:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777047.904132:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904133:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.904135:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904136:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777047.904138:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777047.904140:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777047.904143:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777047.904145:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777047.904149:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777047.904156:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827328:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777047.904160:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777047.904164:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904170:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777047.904171:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.904174:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777047.904175:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777047.904177:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777047.904179:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827456:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777047.904182:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777047.904185:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777047.904187:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.904189:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777047.904193:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777047.904194:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777047.904197:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777047.904200:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777047.904203:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777047.904205:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099827456, portal 4 00000100:00000001:0.0:1775777047.904207:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777047.904208:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287598336 : -114841421953280 : ffff978d65d6c700) 00000100:00000040:0.0:1775777047.904213:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904217:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777047.904219:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777047.904221:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777047.904223:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099827456, offset 0 00000400:00000200:0.0:1775777047.904226:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777047.904229:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.904232:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777047.904234:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777047.904237:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777047.904239:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777047.904241:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777047.904243:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4695:4695:256:4294967295] 192.168.201.139@tcp LPNI seq info [4695:4695:8:4294967295] 00000400:00000200:0.0:1775777047.904247:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.904252:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777047.904271:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.904274:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777047.904277:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777047.904279:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775777047.904283:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777047.904285:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775777047.904288:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.904289:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777047.904292:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777047.904295:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777047.904298:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777047.904299:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777047.904301:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777047.904305:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904311:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.904314:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904318:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.904320:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777047.904323:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777047.904325:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777047.904328:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777047.904330:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313435666 00000100:00000001:0.0:1775777047.904331:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313436920 00000100:00000001:0.0:1775777047.904333:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313435666 00000100:00000001:0.0:1775777047.904335:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313436920 00000100:00000001:0.0:1775777047.904336:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777047.904338:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.904339:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777047.904342:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777047/real 1775777047] req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777047.904349:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777047.904350:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777047.904351:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777047.904353:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777047.904354:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777047.904356:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777047.904358:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777047.904365:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904370:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777047.904371:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904373:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.904374:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777047.904377:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904383:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777047.904386:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904393:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777047.904395:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904405:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.904407:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904409:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777047.904410:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777047.904413:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777047.904416:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777047.904418:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777047.904422:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777047.904430:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827456:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777047.904434:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777047.904438:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904443:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777047.904444:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.904446:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777047.904448:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777047.904449:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777047.904451:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827584:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777047.904454:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777047.904456:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777047.904458:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777047.904459:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777047.904462:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777047.904464:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777047.904467:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777047.904469:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777047.904472:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777047.904475:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099827584, portal 4 00000100:00000001:0.0:1775777047.904477:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777047.904478:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609088 : -114841421942528 : ffff978d65d6f100) 00000100:00000040:0.0:1775777047.904484:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904489:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777047.904491:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777047.904493:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777047.904495:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099827584, offset 0 00000400:00000200:0.0:1775777047.904499:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777047.904503:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.904506:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777047.904509:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777047.904511:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777047.904513:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777047.904515:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777047.904517:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4696:4696:256:4294967295] 192.168.201.139@tcp LPNI seq info [4696:4696:8:4294967295] 00000400:00000200:0.0:1775777047.904520:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777047.904526:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777047.904529:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.904532:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777047.904536:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777047.904538:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775777047.904541:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777047.904544:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775777047.904547:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777047.904548:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777047.904551:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777047.904554:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777047.904556:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777047.904557:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777047.904559:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777047.904563:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904569:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.904571:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904576:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.904577:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777047.904580:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777047.904583:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777047.904586:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777047.904588:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313435657 00000100:00000001:0.0:1775777047.904590:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313436920 00000100:00000001:0.0:1775777047.904592:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313435657 00000100:00000001:0.0:1775777047.904593:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313436920 00000100:00000001:0.0:1775777047.904595:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777047.904597:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.904598:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777047.904601:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777047/real 1775777047] req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777047.904607:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777047.904609:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777047.904610:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777047.904612:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777047.904614:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777047.904615:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777047.904618:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777047.904625:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904629:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777047.904630:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904632:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.904633:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777047.904635:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904641:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777047.904644:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904649:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777047.904652:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904654:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777047.904655:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904656:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777047.904658:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777047.904660:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777047.904663:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777047.904665:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777047.904668:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777047.904675:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827584:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777047.904679:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904680:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.904683:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6fb80 x1862033099827072/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904690:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777047.904692:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777047.904695:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f000. 02000000:00000001:0.0:1775777047.904698:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777047.904701:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777047.904703:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777047.904705:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777047.904707:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38400. 02000000:00000001:0.0:1775777047.904711:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777047.904712:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777047.904714:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6fb80. 00000100:00000001:0.0:1775777047.904718:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777047.904719:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904720:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.904723:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6dc00 x1862033099827200/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904735:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777047.904736:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777047.904737:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c400. 02000000:00000001:0.0:1775777047.904739:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777047.904741:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777047.904742:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777047.904743:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777047.904745:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39800. 02000000:00000001:0.0:1775777047.904747:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777047.904748:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777047.904749:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6dc00. 00000100:00000001:0.0:1775777047.904751:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777047.904753:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904754:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.904757:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c380 x1862033099827328/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904763:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777047.904764:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777047.904765:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759fc00. 02000000:00000001:0.0:1775777047.904767:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777047.904768:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777047.904769:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777047.904770:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777047.904771:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39400. 02000000:00000001:0.0:1775777047.904773:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777047.904774:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777047.904775:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c380. 00000100:00000001:0.0:1775777047.904776:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777047.904806:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904808:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.904812:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c700 x1862033099827456/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904818:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777047.904819:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777047.904821:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f400. 02000000:00000001:0.0:1775777047.904823:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777047.904824:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777047.904825:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777047.904827:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777047.904828:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a400. 02000000:00000001:0.0:1775777047.904830:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777047.904831:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777047.904832:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c700. 00000100:00000001:0.0:1775777047.904834:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777047.904835:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904836:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777047.904839:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f100 x1862033099827584/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777047.904844:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777047.904845:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777047.904846:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e400. 02000000:00000001:0.0:1775777047.904848:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777047.904850:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777047.904851:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777047.904852:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777047.904853:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b800. 02000000:00000001:0.0:1775777047.904855:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777047.904856:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777047.904857:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f100. 00000100:00000001:0.0:1775777047.904859:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777047.904860:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904862:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777047.904871:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777047.904872:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777047.904875:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777047.904876:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777048.092007:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777048.092015:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777048.092017:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777048.092023:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777048.092027:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777048.092032:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777048.092035:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777048.092038:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777048.092041:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777048.603860:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777048.603868:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777048.603870:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777048.603875:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777048.603879:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777048.603883:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777048.603886:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777048.603888:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777048.603891:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777048.671190:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777048.671198:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777048.671201:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777048.671204:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777048.671209:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777048.671212:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777048.671214:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777048.671216:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777048.671220:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777048.671221:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777048.671225:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777048.671227:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777048.671230:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777048.671231:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777048.672117:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777048.672121:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777048.672125:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777049.116079:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777049.116086:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777049.116089:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777049.116094:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777049.116098:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777049.116101:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777049.116104:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777049.116107:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777049.116111:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777049.179919:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313436920, at 4313436928 00000100:00000010:0.0:1775777049.179935:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775777049.180031:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313436920, at 4313436928 00000100:00000010:1.0:1775777049.180037:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237b40. 00000100:00000001:3.0:1775777049.628368:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777049.628377:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777049.628379:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777049.628386:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777049.628390:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777049.628405:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777049.628408:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777049.628410:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777049.628413:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777049.691931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777049.691945:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777049.691952:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777049.691953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777049.691957:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777049.691958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777049.691961:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777049.691962:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777049.692058:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777049.692061:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777049.692065:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777050.140166:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777050.140174:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777050.140176:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777050.140182:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777050.140185:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777050.140189:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777050.140191:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777050.140194:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777050.140198:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777050.653744:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777050.653751:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777050.653753:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777050.653758:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777050.653762:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777050.653766:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777050.653769:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777050.653771:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777050.653776:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777050.716577:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777050.716583:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777050.716589:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777050.716591:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777050.716594:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777050.716595:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777050.716599:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777050.716600:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777050.716810:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777050.716814:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777050.716817:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777051.166763:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777051.166771:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777051.166774:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777051.166814:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777051.166818:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777051.166822:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777051.166825:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777051.166827:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777051.166830:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777051.676085:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777051.676092:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777051.676094:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777051.676099:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777051.676103:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777051.676106:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777051.676109:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777051.676111:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777051.676114:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741214:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777051.741220:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741225:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777051.741229:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777051.741231:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777051.741236:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741238:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777051.741241:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741242:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741254:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741258:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777051.741260:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777051.741261:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777051.741263:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741264:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777051.741265:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741267:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741269:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741273:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777051.741274:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777051.741275:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777051.741276:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741278:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777051.741279:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741280:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777051.741282:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777051.741381:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777051.741384:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777051.741387:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777052.187903:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777052.187911:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777052.187913:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777052.187919:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777052.187923:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777052.187927:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777052.187929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777052.187931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777052.187934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777052.701530:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777052.701536:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777052.701538:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777052.701544:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777052.701547:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777052.701551:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777052.701553:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777052.701556:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777052.701559:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777052.763914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777052.763920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777052.763925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777052.763927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777052.763930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777052.763931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777052.763934:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777052.763935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777052.764036:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777052.764039:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777052.764042:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775777053.026880:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777053.026889:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775777053.026892:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777053.026897:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777053.026902:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777053.026908:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18768 00000400:00000200:0.0:1775777053.026918:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777053.026924:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777053.026927:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777053.026930:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.026933:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.026936:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.026938:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777053.026942:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777053.026944:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775777053.026946:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777053.026949:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775777053.026951:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775777053.026953:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775777053.026956:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.026958:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.026960:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.026962:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777053.026964:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775777053.026968:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777053.026971:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777053.026973:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777053.026977:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777053.026979:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.026981:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775777053.026984:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777053.026992:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454ddc00. 00000020:00000040:0.0:1775777053.026996:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775777053.027000:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777053.027001:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027004:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777053.027007:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777053.027011:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777053.027019:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e400. 00000100:00000001:0.0:1775777053.027025:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777053.027033:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777053.027043:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777053.027047:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777053.027438:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f803cc8] to pc [ptlrpcd_rcv+-1] req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777053.027463:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777053.027467:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777053.027470:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775777053.027471:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777053.027473:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777053.027476:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777053.027479:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18768 00000400:00000200:0.0:1775777053.027483:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777053.027487:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777053.027489:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777053.027492:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027494:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.027496:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027498:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777053.027500:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777053.027501:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775777053.027503:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777053.027505:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775777053.027507:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775777053.027509:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775777053.027512:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027514:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.027515:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027517:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777053.027519:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775777053.027521:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027523:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777053.027525:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777053.027532:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777053.027533:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027536:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777053.027539:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775777053.027542:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027544:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777053.027548:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dc000. 00000020:00000040:0.0:1775777053.027551:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775777053.027553:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777053.027554:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027556:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777053.027557:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777053.027560:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777053.027564:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f400. 00000100:00000001:0.0:1775777053.027568:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777053.027573:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777053.027581:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777053.027583:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777053.027587:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029c6c58f] to pc [ptlrpcd_rcv+-1] req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777053.027594:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777053.027596:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777053.027600:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775777053.027602:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777053.027603:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777053.027606:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777053.027608:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18768 00000400:00000200:0.0:1775777053.027612:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777053.027615:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777053.027617:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777053.027620:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027622:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.027623:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027625:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777053.027627:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777053.027628:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775777053.027630:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777053.027632:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775777053.027633:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775777053.027635:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775777053.027638:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027639:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.027641:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027643:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777053.027645:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775777053.027647:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027649:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777053.027650:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777053.027653:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777053.027655:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027657:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777053.027659:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775777053.027661:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027662:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777053.027666:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775777053.027668:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775777053.027671:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777053.027672:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027674:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777053.027675:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777053.027677:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777053.027680:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759fc00. 00000100:00000001:0.0:1775777053.027683:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777053.027687:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777053.027693:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777053.027695:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777053.027697:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777053.027703:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777053.027706:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777053.027708:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775777053.027709:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777053.027711:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777053.027713:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777053.027715:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18768 00000400:00000200:0.0:1775777053.027718:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777053.027721:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777053.027723:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777053.027726:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027728:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.027730:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027731:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777053.027734:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777053.027735:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775777053.027736:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777053.027738:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775777053.027740:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775777053.027742:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775777053.027745:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027747:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.027748:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027750:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777053.027752:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775777053.027754:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027756:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777053.027758:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777053.027760:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777053.027762:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027763:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777053.027766:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775777053.027768:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027769:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777053.027773:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775777053.027775:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775777053.027814:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777053.027816:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027818:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777053.027819:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777053.027821:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777053.027825:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c400. 00000100:00000001:0.0:1775777053.027828:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777053.027832:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777053.027839:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777053.027841:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777053.027844:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777053.027850:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777053.027853:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777053.027856:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775777053.027857:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777053.027858:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777053.027861:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777053.027863:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18768 00000400:00000200:0.0:1775777053.027867:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777053.027870:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777053.027872:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777053.027875:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027877:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.027878:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027880:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777053.027882:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777053.027883:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775777053.027885:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777053.027887:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775777053.027888:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775777053.027890:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775777053.027893:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027894:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777053.027896:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027898:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777053.027900:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775777053.027902:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777053.027905:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777053.027906:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777053.027909:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777053.027911:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027912:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777053.027914:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775777053.027916:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027918:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777053.027921:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775777053.027923:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775777053.027926:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777053.027927:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777053.027928:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777053.027930:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777053.027931:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777053.027934:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f000. 00000100:00000001:0.0:1775777053.027938:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777053.027942:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777053.027949:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777053.027951:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777053.027955:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777053.027961:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777053.027965:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18778) 00000100:00000001:3.0:1775777053.028030:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775777053.028035:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777053.028037:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777053.028042:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.028049:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777053.028063:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.028067:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777053.028069:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777053.028072:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777053.028075:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099827712:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775777053.028079:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777053.028082:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777053.028084:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.028086:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777053.028091:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775777053.028093:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777053.028098:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777053.028102:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777053.028108:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777053.028116:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099827712, portal 25 00000100:00000001:3.0:1775777053.028118:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777053.028119:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741754368 : -114841967797248 : ffff978d454ddc00) 00000100:00000040:3.0:1775777053.028124:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.028131:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777053.028134:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777053.028137:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777053.028140:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099827712, offset 0 00000400:00000200:3.0:1775777053.028146:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777053.028155:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.028161:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777053.028178:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777053.028184:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777053.028189:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777053.028193:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777053.028197:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4697:4697:256:4294967295] 192.168.201.139@tcp LPNI seq info [4697:4697:8:4294967295] 00000400:00000200:3.0:1775777053.028201:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.028211:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777053.028217:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.028222:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777053.028229:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777053.028232:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775777053.028238:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777053.028240:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775777053.028245:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.028247:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777053.028253:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777053.028258:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777053.028262:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777053.028264:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777053.028268:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777053.028283:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.028292:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.028295:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.028302:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.028304:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777053.028310:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777053.028314:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777053.028316:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313442045 00000100:00000010:3.0:1775777053.028321:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344ba0. 00000100:00000001:3.0:1775777053.028328:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775777053.028335:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313442045 00000100:00000010:3.0:1775777053.028969:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d50344d20. 00000100:00000001:3.0:1775777053.028973:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775777053.028978:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777053.028982:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.028984:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777053.028990:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777053/real 1775777053] req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777053.029001:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777053.029002:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777053.029005:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777053.029008:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777053.029009:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777053.029012:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777053.029015:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777053.029024:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029031:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777053.029032:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.029034:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.029036:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777053.029039:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029046:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777053.029063:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029071:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777053.029074:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.029075:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.029076:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.029078:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777053.029081:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777053.029083:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777053.029087:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777053.029089:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777053.029093:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777053.029101:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099827712:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775777053.029106:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777053.029109:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029116:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777053.029117:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.029120:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777053.029123:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777053.029125:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777053.029127:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827840:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775777053.029130:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777053.029132:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777053.029134:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.029136:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777053.029139:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159000. 02000000:00000001:3.0:1775777053.029141:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777053.029144:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777053.029147:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777053.029150:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777053.029154:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099827840, portal 10 00000100:00000001:3.0:1775777053.029155:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777053.029157:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741747200 : -114841967804416 : ffff978d454dc000) 00000100:00000040:3.0:1775777053.029163:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029508:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777053.029511:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777053.029515:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777053.029520:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099827840, offset 0 00000400:00000200:3.0:1775777053.029525:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777053.029531:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.029537:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777053.029540:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777053.029544:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777053.029546:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777053.029549:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777053.029552:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4698:4698:256:4294967295] 192.168.201.139@tcp LPNI seq info [4698:4698:8:4294967295] 00000400:00000200:3.0:1775777053.029556:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.029564:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777053.029569:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.029573:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777053.029578:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777053.029582:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775777053.029586:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777053.029589:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775777053.029592:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.029594:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777053.029598:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777053.029602:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777053.029606:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777053.029608:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777053.029610:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777053.029614:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029622:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.029625:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029636:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.029638:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777053.029641:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777053.029646:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:3.0:1775777053.029649:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777053.029651:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313440781 00000100:00000001:3.0:1775777053.029654:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313442045 00000100:00000001:3.0:1775777053.029656:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313440781 00000100:00000001:3.0:1775777053.029658:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313442045 00000100:00000001:3.0:1775777053.029660:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777053.029662:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.029664:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777053.029668:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777053/real 1775777053] req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777053.029676:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777053.029677:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777053.029679:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777053.029681:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777053.029682:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777053.029684:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777053.029687:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777053.029695:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029701:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777053.029702:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.029704:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.029705:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777053.029708:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029715:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777053.029719:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029725:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777053.029727:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.029728:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.029730:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.029731:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777053.029734:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777053.029736:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777053.029740:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777053.029742:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777053.029746:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777053.029753:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827840:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775777053.029758:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777053.029761:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029769:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777053.029771:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.029772:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777053.029774:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777053.029814:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777053.029817:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827968:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775777053.029820:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777053.029823:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777053.029825:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.029827:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777053.029830:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775777053.029832:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777053.029835:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777053.029838:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777053.029841:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777053.029844:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099827968, portal 10 00000100:00000001:3.0:1775777053.029845:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777053.029847:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:3.0:1775777053.029852:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029859:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777053.029861:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777053.029863:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777053.029865:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099827968, offset 0 00000400:00000200:3.0:1775777053.029869:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777053.029873:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.029876:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777053.029879:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777053.029881:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777053.029883:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777053.029885:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777053.029888:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4699:4699:256:4294967295] 192.168.201.139@tcp LPNI seq info [4699:4699:8:4294967295] 00000400:00000200:3.0:1775777053.029891:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.029896:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777053.029900:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.029903:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777053.029907:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777053.029909:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775777053.029913:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777053.029915:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775777053.029919:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.029921:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777053.029924:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777053.029927:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777053.029930:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777053.029932:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777053.029934:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777053.029937:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029945:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.029947:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.029952:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.029954:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777053.029957:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777053.029959:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:3.0:1775777053.029962:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777053.029964:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313440783 00000100:00000001:3.0:1775777053.029966:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313442045 00000100:00000001:3.0:1775777053.029967:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313440783 00000100:00000001:3.0:1775777053.029969:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313442045 00000100:00000001:3.0:1775777053.029971:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777053.029973:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.029975:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777053.029978:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777053/real 1775777053] req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777053.029985:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777053.029987:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777053.029988:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777053.029990:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777053.029992:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777053.029994:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777053.029996:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777053.030003:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.030009:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777053.030010:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.030011:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.030012:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777053.030015:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.030021:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777053.030024:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.030030:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777053.030032:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.030033:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.030034:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.030036:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777053.030038:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777053.030040:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777053.030042:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777053.030044:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777053.030047:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777053.031468:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099827968:192.168.201.139@tcp:38:kworker.0 00000100:00000001:3.0:1775777053.031476:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777053.031483:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031491:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777053.031493:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.031495:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777053.031498:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777053.031501:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777053.031504:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828096:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775777053.031508:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777053.031511:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777053.031513:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.031515:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777053.031524:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775777053.031526:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777053.031533:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777053.031537:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777053.031543:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777053.031547:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099828096, portal 4 00000100:00000001:3.0:1775777053.031550:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777053.031551:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:3.0:1775777053.031558:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031564:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777053.031566:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777053.031568:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777053.031570:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099828096, offset 0 00000400:00000200:3.0:1775777053.031576:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777053.031582:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.031588:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777053.031592:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777053.031596:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777053.031598:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777053.031601:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777053.031604:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4700:4700:256:4294967295] 192.168.201.139@tcp LPNI seq info [4700:4700:8:4294967295] 00000400:00000200:3.0:1775777053.031608:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.031617:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777053.031622:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.031626:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777053.031631:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777053.031634:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775777053.031639:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777053.031642:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775777053.031662:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.031664:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777053.031670:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777053.031674:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777053.031679:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777053.031681:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777053.031684:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777053.031689:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031696:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.031699:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031706:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.031707:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777053.031711:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777053.031715:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:3.0:1775777053.031718:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777053.031720:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313440793 00000100:00000001:3.0:1775777053.031723:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313442045 00000100:00000001:3.0:1775777053.031725:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313440793 00000100:00000001:3.0:1775777053.031727:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313442045 00000100:00000001:3.0:1775777053.031729:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777053.031732:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.031734:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777053.031737:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777053/real 1775777053] req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777053.031745:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777053.031747:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777053.031748:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777053.031751:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777053.031752:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777053.031754:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777053.031756:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777053.031763:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031769:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777053.031770:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.031772:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.031774:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777053.031793:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031799:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777053.031803:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031809:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777053.031811:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.031812:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.031814:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.031816:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777053.031818:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777053.031820:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777053.031824:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777053.031826:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777053.031829:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777053.031836:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828096:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775777053.031839:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777053.031842:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031847:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777053.031849:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.031851:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777053.031853:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777053.031854:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777053.031857:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828224:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775777053.031860:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777053.031862:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777053.031863:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777053.031865:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775777053.031868:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775777053.031870:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775777053.031873:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:3.0:1775777053.031875:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d5063d4d0. 00000400:00000010:3.0:1775777053.031878:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1440. 00000100:00000200:3.0:1775777053.031881:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099828224, portal 4 00000100:00000001:3.0:1775777053.031882:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777053.031883:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:3.0:1775777053.031888:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031894:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775777053.031896:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:3.0:1775777053.031898:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d454e1dd0. 00000100:00000200:3.0:1775777053.031900:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099828224, offset 0 00000400:00000200:3.0:1775777053.031904:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:3.0:1775777053.031907:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.031910:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:3.0:1775777053.031913:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777053.031915:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777053.031917:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775777053.031919:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:3.0:1775777053.031922:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4701:4701:256:4294967295] 192.168.201.139@tcp LPNI seq info [4701:4701:8:4294967295] 00000400:00000200:3.0:1775777053.031926:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777053.031931:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:3.0:1775777053.031935:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.031938:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d6f511900. 00000800:00000200:3.0:1775777053.031941:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.0:1775777053.031944:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775777053.031948:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:3.1:1775777053.031950:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1775777053.031953:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:3.0:1775777053.031955:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d6f511900. 00000400:00000200:3.0:1775777053.031957:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1775777053.031960:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1775777053.031963:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1dd0 00000400:00000010:3.0:1775777053.031965:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1dd0. 00000100:00000001:3.0:1775777053.031967:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775777053.031970:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031976:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.031979:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.031985:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.031986:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775777053.031989:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777053.031992:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:3.0:1775777053.031995:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:3.0:1775777053.031997:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313440785 00000100:00000001:3.0:1775777053.031999:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313442045 00000100:00000001:3.0:1775777053.032001:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313440785 00000100:00000001:3.0:1775777053.032003:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313442045 00000100:00000001:3.0:1775777053.032004:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777053.032006:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.032008:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1775777053.032011:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777053/real 1775777053] req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1775777053.032019:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1775777053.032020:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1775777053.032022:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d5063d4d0. 00000400:00000200:3.0:1775777053.032024:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d454e1440 00000400:00000010:3.0:1775777053.032026:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d454e1440. 00000100:00000001:3.0:1775777053.032028:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1775777053.032031:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1775777053.032037:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.032042:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1775777053.032044:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.032045:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.032046:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775777053.032049:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.036557:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775777053.036574:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.036584:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775777053.036589:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036591:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1775777053.036593:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036596:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1775777053.036599:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1775777053.036602:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775777053.036607:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:3.0:1775777053.036610:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775777053.036614:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775777053.036622:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828224:192.168.201.139@tcp:8:kworker.0 00000100:00000001:3.0:1775777053.036627:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036631:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.036635:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454ddc00 x1862033099827712/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.036641:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777053.036644:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777053.036650:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d5115b400. 02000000:00000001:3.0:1775777053.036655:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777053.036658:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777053.036660:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:3.0:1775777053.036662:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777053.036665:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e400. 02000000:00000001:3.0:1775777053.036670:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777053.036672:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777053.036674:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454ddc00. 00000100:00000001:3.0:1775777053.036678:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777053.036679:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036681:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.036686:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dc000 x1862033099827840/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.036691:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777053.036692:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777053.036694:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159000. 02000000:00000001:3.0:1775777053.036696:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777053.036697:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777053.036698:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:3.0:1775777053.036700:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777053.036701:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f400. 02000000:00000001:3.0:1775777053.036703:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777053.036704:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777053.036706:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dc000. 00000100:00000001:3.0:1775777053.036708:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777053.036709:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036710:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.036713:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099827968/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.036719:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777053.036721:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777053.036722:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158c00. 02000000:00000001:3.0:1775777053.036724:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777053.036725:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777053.036726:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:3.0:1775777053.036727:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777053.036728:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759fc00. 02000000:00000001:3.0:1775777053.036730:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777053.036731:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777053.036732:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:3.0:1775777053.036734:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777053.036736:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036737:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.036741:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099828096/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.036752:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777053.036753:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777053.036754:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51158400. 02000000:00000001:3.0:1775777053.036756:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777053.036757:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777053.036758:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:3.0:1775777053.036760:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777053.036761:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c400. 02000000:00000001:3.0:1775777053.036763:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777053.036764:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777053.036766:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:3.0:1775777053.036768:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777053.036769:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036770:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777053.036773:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099828224/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777053.036799:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1775777053.036800:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1775777053.036802:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51159800. 02000000:00000001:3.0:1775777053.036804:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1775777053.036805:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1775777053.036806:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:3.0:1775777053.036808:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1775777053.036809:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f000. 02000000:00000001:3.0:1775777053.036811:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1775777053.036813:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1775777053.036814:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:3.0:1775777053.036816:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1775777053.036817:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036820:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.036829:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777053.036831:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.036835:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775777053.036836:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.211889:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777053.211896:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777053.211898:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777053.211904:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777053.211907:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777053.211911:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.211914:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777053.211916:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.211919:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777053.723904:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777053.723912:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777053.723913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777053.723918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777053.723922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777053.723926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.723929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777053.723931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777053.723934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777053.788662:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777053.788668:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777053.788674:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777053.788675:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777053.788679:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777053.788680:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777053.788683:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777053.788684:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777053.795000:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777053.795006:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777053.795012:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777054.236029:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777054.236037:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777054.236039:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777054.236044:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777054.236049:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777054.236053:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777054.236056:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777054.236059:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777054.236062:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777054.299902:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313442045, at 4313442048 00000100:00000010:0.0:1775777054.299920:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344ba0. 00000100:00000040:1.0:1775777054.301949:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313442045, at 4313442050 00000100:00000010:1.0:1775777054.301962:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d50344d20. 00000100:00000001:3.0:1775777054.747902:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777054.747911:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777054.747913:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777054.747918:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777054.747922:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777054.747926:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777054.747929:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777054.747931:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777054.747934:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777054.811908:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777054.811915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777054.811920:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777054.811921:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777054.811925:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777054.811926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777054.811930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777054.811931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777054.812033:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777054.812037:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777054.812040:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777055.261833:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777055.261840:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777055.261842:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777055.261848:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777055.261853:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777055.261856:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777055.261859:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777055.261861:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777055.261875:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777055.771941:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777055.771949:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777055.771951:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777055.771957:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777055.771961:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777055.771966:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777055.771968:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777055.771971:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777055.771974:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777055.836913:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777055.836920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777055.836926:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777055.836927:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777055.836930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777055.836931:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777055.836936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777055.836938:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777055.837039:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777055.837042:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777055.837046:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777056.289630:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777056.289636:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777056.289639:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777056.289644:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777056.289648:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777056.289653:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777056.289655:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777056.289657:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777056.289660:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777056.795857:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777056.795864:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777056.795865:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777056.795870:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777056.795874:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777056.795878:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777056.795880:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777056.795882:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777056.795885:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777056.863157:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777056.863163:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777056.863169:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777056.863170:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777056.863173:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777056.863175:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777056.863177:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777056.863179:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777056.863616:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777056.863619:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777056.863623:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777057.308363:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777057.308370:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777057.308372:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777057.308377:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777057.308380:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777057.308384:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777057.308387:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777057.308400:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777057.308403:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777057.819926:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777057.819932:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777057.819934:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777057.819939:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777057.819943:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777057.819948:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777057.819950:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777057.819952:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777057.819955:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777057.883918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777057.883923:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777057.883929:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777057.883930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777057.883933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777057.883935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777057.883937:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777057.883939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777057.884031:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777057.884034:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777057.884037:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775777058.140462:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777058.140472:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775777058.140475:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777058.140480:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777058.140484:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777058.140491:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18773 00000400:00000200:3.0:1775777058.140502:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777058.140509:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777058.140512:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777058.140516:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140519:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.140522:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140524:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777058.140531:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777058.140533:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775777058.140536:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777058.140539:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775777058.140541:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775777058.140544:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775777058.140547:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140550:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.140552:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140554:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777058.140557:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775777058.140560:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140563:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777058.140566:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777058.140570:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777058.140573:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140575:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775777058.140577:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777058.140587:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45ded500. 00000020:00000040:3.0:1775777058.140591:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775777058.140596:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777058.140598:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140601:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777058.140603:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777058.140607:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777058.140615:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159800. 00000100:00000001:3.0:1775777058.140622:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777058.140632:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777058.140641:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777058.140646:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777058.140650:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000975f3dd0] to pc [ptlrpcd_rcv+-1] req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777058.140684:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777058.140687:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777058.140690:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775777058.140691:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777058.140692:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777058.140695:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777058.140697:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18773 00000400:00000200:3.0:1775777058.140701:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777058.140705:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777058.140707:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777058.140709:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140711:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.140712:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140714:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777058.140716:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777058.140717:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775777058.140718:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777058.140720:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775777058.140722:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775777058.140723:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775777058.140726:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140728:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.140729:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140730:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777058.140732:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775777058.140734:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140736:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777058.140737:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777058.140742:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777058.140744:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140745:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777058.140749:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777058.140751:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140753:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777058.140756:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec000. 00000020:00000040:3.0:1775777058.140758:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775777058.140760:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777058.140762:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140764:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777058.140765:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777058.140766:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777058.140770:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158400. 00000100:00000001:3.0:1775777058.140772:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777058.140776:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777058.140810:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777058.140812:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777058.140816:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007336e28f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777058.140824:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777058.140826:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777058.140829:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775777058.140830:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777058.140831:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777058.140834:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777058.140836:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18773 00000400:00000200:3.0:1775777058.140840:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777058.140843:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777058.140845:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777058.140847:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140849:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.140851:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140853:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777058.140855:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777058.140856:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775777058.140858:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777058.140859:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775777058.140861:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775777058.140864:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775777058.140867:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140868:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.140870:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140872:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777058.140874:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775777058.140877:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140879:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777058.140880:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777058.140883:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777058.140885:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140887:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777058.140888:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777058.140890:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140892:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777058.140895:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec380. 00000020:00000040:3.0:1775777058.140898:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775777058.140900:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777058.140901:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140903:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777058.140904:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777058.140906:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777058.140909:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158c00. 00000100:00000001:3.0:1775777058.140912:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777058.140917:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777058.140923:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777058.140925:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777058.140929:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ded9add] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777058.140936:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777058.140939:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777058.140941:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775777058.140943:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777058.140944:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777058.140947:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777058.140949:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18773 00000400:00000200:3.0:1775777058.140952:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777058.140955:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777058.140958:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777058.140960:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140962:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.140964:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140966:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777058.140968:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777058.140969:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775777058.140971:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777058.140972:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775777058.140974:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775777058.140976:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775777058.140979:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140981:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.140983:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140984:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777058.140986:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775777058.140988:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777058.140990:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777058.140991:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777058.140993:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777058.140995:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.140997:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777058.141000:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777058.141002:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.141004:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777058.141007:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deca80. 00000020:00000040:3.0:1775777058.141010:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775777058.141012:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777058.141013:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.141015:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777058.141016:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777058.141017:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777058.141021:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159000. 00000100:00000001:3.0:1775777058.141024:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777058.141028:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777058.141034:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777058.141036:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777058.141039:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000584754af] to pc [ptlrpcd_rcv+-1] req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777058.141046:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777058.141048:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777058.141051:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775777058.141053:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777058.141054:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777058.141056:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777058.141058:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18773 00000400:00000200:3.0:1775777058.141062:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777058.141065:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777058.141067:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777058.141070:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.141072:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.141073:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.141107:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777058.141110:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777058.141111:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775777058.141113:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777058.141115:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775777058.141116:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775777058.141118:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775777058.141121:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.141123:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777058.141125:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777058.141127:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777058.141129:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775777058.141131:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777058.141133:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777058.141135:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777058.141138:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777058.141140:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.141141:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777058.141143:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777058.141145:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.141146:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777058.141150:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dee680. 00000020:00000040:3.0:1775777058.141152:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775777058.141154:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777058.141155:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.141157:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777058.141158:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777058.141159:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777058.141162:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b400. 00000100:00000001:3.0:1775777058.141164:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777058.141168:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777058.141179:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777058.141181:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777058.141337:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dc2ce869] to pc [ptlrpcd_rcv+-1] req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777058.141345:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777058.141350:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18783) 00000100:00000001:0.0:1775777058.141425:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777058.141430:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777058.141432:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777058.141439:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141447:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777058.141448:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.141452:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777058.141454:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777058.141457:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777058.141460:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099828352:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777058.141463:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777058.141467:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777058.141469:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.141472:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777058.141476:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777058.141478:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777058.141484:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777058.141489:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777058.141495:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777058.141503:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099828352, portal 25 00000100:00000001:0.0:1775777058.141505:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777058.141506:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751255296 : -114841958296320 : ffff978d45ded500) 00000100:00000040:0.0:1775777058.141512:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141519:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777058.141521:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777058.141523:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777058.141526:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099828352, offset 0 00000400:00000200:0.0:1775777058.141532:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777058.141540:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.141546:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777058.141551:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777058.141555:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777058.141559:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777058.141562:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777058.141566:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4702:4702:256:4294967295] 192.168.201.139@tcp LPNI seq info [4702:4702:8:4294967295] 00000400:00000200:0.0:1775777058.141570:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.141579:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777058.141584:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.141589:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777058.141596:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777058.141600:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775777058.141604:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777058.141608:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775777058.141612:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.141614:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777058.141621:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777058.141626:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777058.141631:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777058.141633:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777058.141638:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777058.141643:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141652:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.141654:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141660:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.141662:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777058.141680:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777058.141685:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777058.141687:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313447158 00000100:00000010:0.0:1775777058.141692:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777058.141698:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777058.141705:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313447158 00000100:00000010:0.0:1775777058.141707:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237d20. 00000100:00000001:0.0:1775777058.141710:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777058.141713:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777058.141716:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.141718:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777058.141722:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777058/real 1775777058] req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777058.141732:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777058.141734:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777058.141736:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777058.141739:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777058.141741:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777058.141743:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777058.141747:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777058.141754:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141765:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777058.141767:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.141770:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.141771:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777058.141775:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141805:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777058.141809:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141816:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777058.141818:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.141820:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.141821:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.141823:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777058.141826:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777058.141828:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777058.141832:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777058.141835:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777058.141838:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777058.141845:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099828352:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777058.141850:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777058.141853:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141859:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777058.141860:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.141862:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777058.141864:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777058.141866:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777058.141869:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828480:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777058.141872:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777058.141874:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777058.141875:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.141877:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777058.141880:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777058.141882:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777058.141885:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777058.141888:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777058.141892:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777058.141895:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099828480, portal 10 00000100:00000001:0.0:1775777058.141897:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777058.141898:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751249920 : -114841958301696 : ffff978d45dec000) 00000100:00000040:0.0:1775777058.141904:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141910:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777058.141912:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777058.141914:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777058.141917:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099828480, offset 0 00000400:00000200:0.0:1775777058.141921:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777058.141925:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.141928:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777058.141931:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777058.141934:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777058.141936:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777058.141937:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777058.141940:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4703:4703:256:4294967295] 192.168.201.139@tcp LPNI seq info [4703:4703:8:4294967295] 00000400:00000200:0.0:1775777058.141943:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.141948:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777058.141952:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.141955:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777058.141959:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777058.141961:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775777058.141965:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777058.141967:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775777058.141970:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.141971:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777058.141974:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777058.141977:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777058.141980:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777058.141981:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777058.141983:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777058.141987:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.141994:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.141996:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142001:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.142002:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777058.142005:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777058.142008:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777058.142011:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777058.142012:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313445894 00000100:00000001:0.0:1775777058.142015:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313447158 00000100:00000001:0.0:1775777058.142016:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313445894 00000100:00000001:0.0:1775777058.142018:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313447158 00000100:00000001:0.0:1775777058.142020:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777058.142022:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.142023:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777058.142027:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777058/real 1775777058] req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777058.142038:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777058.142039:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777058.142040:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777058.142042:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777058.142044:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777058.142046:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777058.142048:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777058.142056:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142061:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777058.142063:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142064:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.142065:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777058.142068:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142074:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777058.142086:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142093:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777058.142095:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142097:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.142098:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142099:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777058.142101:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777058.142103:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777058.142106:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777058.142108:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777058.142112:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777058.142119:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828480:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777058.142122:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777058.142124:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142131:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777058.142132:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.142134:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777058.142136:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777058.142137:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777058.142140:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828608:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777058.142143:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777058.142145:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777058.142147:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.142148:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777058.142151:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775777058.142153:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777058.142156:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777058.142158:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777058.142160:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777058.142164:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099828608, portal 10 00000100:00000001:0.0:1775777058.142165:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777058.142166:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751250816 : -114841958300800 : ffff978d45dec380) 00000100:00000040:0.0:1775777058.142171:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142177:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777058.142179:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777058.142321:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777058.142326:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099828608, offset 0 00000400:00000200:0.0:1775777058.142333:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777058.142337:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.142340:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777058.142342:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777058.142344:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777058.142346:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777058.142348:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777058.142351:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4704:4704:256:4294967295] 192.168.201.139@tcp LPNI seq info [4704:4704:8:4294967295] 00000400:00000200:0.0:1775777058.142354:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.142359:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777058.142363:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.142366:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777058.142370:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777058.142373:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775777058.142376:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777058.142378:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775777058.142382:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.142384:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777058.142387:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777058.142389:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777058.142392:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777058.142393:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777058.142395:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777058.142399:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142405:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.142408:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142413:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.142414:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777058.142417:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777058.142420:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777058.142423:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777058.142425:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313445895 00000100:00000001:0.0:1775777058.142427:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313447158 00000100:00000001:0.0:1775777058.142429:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313445895 00000100:00000001:0.0:1775777058.142431:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313447158 00000100:00000001:0.0:1775777058.142433:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777058.142435:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.142436:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777058.142439:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777058/real 1775777058] req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777058.142446:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777058.142447:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777058.142448:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777058.142450:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777058.142452:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777058.142454:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777058.142457:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777058.142463:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142468:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777058.142470:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142472:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.142473:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777058.142476:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142483:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777058.142487:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142493:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777058.142496:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142497:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.142499:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142500:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777058.142501:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777058.142504:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777058.142508:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777058.142510:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777058.142513:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777058.142520:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828608:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777058.142524:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777058.142527:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142533:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777058.142534:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.142537:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777058.142538:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777058.142540:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777058.142543:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828736:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777058.142546:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777058.142548:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777058.142549:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.142551:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777058.142554:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775777058.142555:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777058.142559:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777058.142561:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777058.142564:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777058.142567:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099828736, portal 4 00000100:00000001:0.0:1775777058.142569:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777058.142570:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751252608 : -114841958299008 : ffff978d45deca80) 00000100:00000040:0.0:1775777058.142576:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142581:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777058.142583:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777058.142585:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777058.142587:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099828736, offset 0 00000400:00000200:0.0:1775777058.142590:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777058.142594:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.142597:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777058.142599:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777058.142601:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777058.142604:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777058.142605:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777058.142608:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4705:4705:256:4294967295] 192.168.201.139@tcp LPNI seq info [4705:4705:8:4294967295] 00000400:00000200:0.0:1775777058.142611:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.142617:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777058.142621:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.142624:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777058.142627:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777058.142630:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775777058.142634:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777058.142636:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775777058.142640:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.142642:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777058.142644:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777058.142647:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777058.142650:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777058.142652:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777058.142654:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777058.142657:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142716:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.142720:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142725:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.142727:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777058.142731:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777058.142735:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777058.142737:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777058.142739:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313445904 00000100:00000001:0.0:1775777058.142741:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313447158 00000100:00000001:0.0:1775777058.142743:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313445904 00000100:00000001:0.0:1775777058.142745:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313447158 00000100:00000001:0.0:1775777058.142747:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777058.142749:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.142751:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777058.142753:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777058/real 1775777058] req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777058.142760:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777058.142761:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777058.142763:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777058.142766:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777058.142767:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777058.142769:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777058.142771:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777058.142808:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142815:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777058.142816:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142818:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.142820:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777058.142823:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142829:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777058.142832:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142838:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777058.142840:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142841:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.142842:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.142844:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777058.142845:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777058.142848:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777058.142850:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777058.142852:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777058.142856:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777058.142863:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828736:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777058.142866:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777058.142870:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142876:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777058.142878:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.142880:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777058.142882:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777058.142883:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777058.142886:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828864:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777058.142889:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777058.142891:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777058.142893:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777058.142894:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777058.142898:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777058.142899:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777058.142902:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777058.142904:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777058.142907:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777058.142910:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099828864, portal 4 00000100:00000001:0.0:1775777058.142912:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777058.142913:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751259776 : -114841958291840 : ffff978d45dee680) 00000100:00000040:0.0:1775777058.142917:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.142922:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777058.142924:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777058.142926:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777058.142928:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099828864, offset 0 00000400:00000200:0.0:1775777058.142931:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777058.142934:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.142937:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777058.142940:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777058.142942:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777058.142944:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777058.142945:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777058.142948:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4706:4706:256:4294967295] 192.168.201.139@tcp LPNI seq info [4706:4706:8:4294967295] 00000400:00000200:0.0:1775777058.142951:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777058.142956:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777058.142960:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.142963:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777058.142967:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777058.142969:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775777058.142973:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777058.142976:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775777058.142979:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777058.142981:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777058.142983:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777058.142986:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777058.142989:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777058.142990:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777058.142993:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777058.142996:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.143003:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.143006:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.143012:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.143013:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777058.143017:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777058.143021:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777058.143024:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777058.143026:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313445896 00000100:00000001:0.0:1775777058.143028:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313447158 00000100:00000001:0.0:1775777058.143030:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313445896 00000100:00000001:0.0:1775777058.143031:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313447158 00000100:00000001:0.0:1775777058.143033:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777058.143035:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.143036:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777058.143039:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777058/real 1775777058] req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777058.143045:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777058.143047:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777058.143048:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777058.143050:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777058.143052:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777058.143054:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777058.143056:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777058.143063:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.143068:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777058.143069:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.143071:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.143072:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777058.143075:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.143094:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777058.143099:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.143105:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777058.143107:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.143109:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777058.143110:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.143112:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777058.143114:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777058.143116:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777058.143120:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777058.143121:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777058.143125:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777058.143138:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099828864:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777058.143142:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.143145:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.143149:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45ded500 x1862033099828352/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.143155:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777058.143157:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777058.143159:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777058.143162:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777058.143165:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777058.143166:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777058.143168:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777058.143170:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159800. 02000000:00000001:0.0:1775777058.143174:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777058.143176:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777058.143177:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45ded500. 00000100:00000001:0.0:1775777058.145335:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777058.145343:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.145348:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.145361:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec000 x1862033099828480/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.145370:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777058.145372:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777058.145376:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777058.145382:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777058.145384:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777058.145386:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777058.145389:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777058.145390:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158400. 02000000:00000001:0.0:1775777058.145394:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777058.145395:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777058.145397:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec000. 00000100:00000001:0.0:1775777058.145401:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777058.145403:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.145404:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.145408:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec380 x1862033099828608/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.145415:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777058.145416:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777058.145417:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775777058.145419:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777058.145421:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777058.145422:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777058.145423:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777058.145425:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158c00. 02000000:00000001:0.0:1775777058.145427:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777058.145428:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777058.145429:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec380. 00000100:00000001:0.0:1775777058.145430:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777058.145431:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.145433:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.145436:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deca80 x1862033099828736/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.145443:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777058.145444:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777058.145445:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775777058.145448:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777058.145449:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777058.145450:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777058.145451:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777058.145452:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159000. 02000000:00000001:0.0:1775777058.145454:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777058.145455:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777058.145456:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deca80. 00000100:00000001:0.0:1775777058.145458:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777058.145459:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.145460:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777058.145464:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dee680 x1862033099828864/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777058.145471:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777058.145472:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777058.145473:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777058.145476:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777058.145477:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777058.145478:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777058.145480:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777058.145481:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b400. 02000000:00000001:0.0:1775777058.145483:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777058.145484:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777058.145485:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dee680. 00000100:00000001:0.0:1775777058.145488:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777058.145489:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.145491:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777058.145499:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777058.145501:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777058.145504:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777058.145506:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.333812:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777058.333818:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777058.333820:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777058.333825:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777058.333828:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777058.333832:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777058.333834:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777058.333836:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777058.333839:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777058.843918:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777058.843925:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777058.843927:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777058.843932:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777058.843936:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777058.843940:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777058.843942:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777058.843945:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777058.843947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777058.907922:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777058.907928:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777058.907930:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777058.907933:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777058.907938:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777058.907940:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777058.907942:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777058.907944:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777058.907948:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777058.907949:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777058.907953:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777058.907954:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777058.907957:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777058.907958:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777058.908189:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777058.908193:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777058.908196:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777059.355929:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777059.355937:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777059.355938:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777059.355945:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777059.355949:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777059.355953:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777059.355955:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777059.355957:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777059.355960:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777059.419886:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313447158, at 4313447168 00000100:00000010:1.0:1775777059.419900:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237d20. 00000100:00000040:0.0:1775777059.419970:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313447158, at 4313447168 00000100:00000010:0.0:1775777059.419976:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775777059.868215:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777059.868222:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777059.868224:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777059.868229:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777059.868238:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777059.868241:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777059.868243:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777059.868245:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777059.868248:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777059.935835:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777059.935841:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777059.935846:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777059.935847:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777059.935851:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777059.935852:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777059.935856:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777059.935857:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777059.935948:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777059.935951:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777059.935954:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777060.380574:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777060.380581:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777060.380583:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777060.380588:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777060.380592:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777060.380596:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777060.380599:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777060.380601:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777060.380604:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777060.892317:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777060.892324:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777060.892326:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777060.892332:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777060.892336:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777060.892340:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777060.892343:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777060.892345:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777060.892348:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777060.955919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777060.955926:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777060.955931:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777060.955932:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777060.955936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777060.955937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777060.955940:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777060.955941:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777060.956049:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777060.956053:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777060.956056:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777061.403903:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777061.403911:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777061.403912:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777061.403918:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777061.403922:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777061.403926:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777061.403929:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777061.403931:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777061.403934:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777061.915914:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777061.915921:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777061.915923:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777061.915929:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777061.915934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777061.915938:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777061.915941:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777061.915943:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777061.915945:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981314:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777061.981320:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981326:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777061.981330:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777061.981332:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777061.981336:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981338:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777061.981341:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981342:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981344:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981349:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777061.981350:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777061.981351:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777061.981352:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981353:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777061.981355:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981356:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981357:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981360:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777061.981361:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777061.981363:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777061.981364:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981365:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777061.981366:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981368:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777061.981369:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777061.981743:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777061.981746:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777061.981749:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777062.428713:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777062.428720:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777062.428722:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777062.428727:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777062.428730:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777062.428734:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777062.428736:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777062.428739:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777062.428742:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777062.940044:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777062.940051:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777062.940053:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777062.940060:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777062.940064:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777062.940068:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777062.940070:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777062.940073:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777062.940076:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777063.004372:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777063.004378:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777063.004386:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777063.004387:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777063.004390:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777063.004392:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777063.004395:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777063.004396:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777063.004498:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777063.004502:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777063.004505:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775777063.260063:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777063.260072:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775777063.260085:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777063.260089:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777063.260104:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777063.260112:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18778 00000400:00000200:3.0:1775777063.260123:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777063.260130:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777063.260134:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777063.260137:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260141:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.260143:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260147:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777063.260151:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777063.260153:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775777063.260156:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777063.260159:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775777063.260161:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775777063.260164:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775777063.260167:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260168:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.260170:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260172:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777063.260175:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775777063.260179:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260182:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777063.260184:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777063.260188:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777063.260191:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.260193:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775777063.260195:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777063.260207:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dece00. 00000020:00000040:3.0:1775777063.260211:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775777063.260215:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777063.260217:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.260220:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777063.260223:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777063.260228:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777063.260237:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159c00. 00000100:00000001:3.0:1775777063.260244:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777063.260254:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777063.260265:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777063.260269:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777063.260275:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003d022cac] to pc [ptlrpcd_rcv+-1] req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.260344:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777063.260349:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777063.260351:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777063.260357:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.260366:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777063.260367:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.260371:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777063.260373:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777063.260376:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777063.260379:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099828992:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777063.260383:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777063.260387:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777063.260390:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.260392:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777063.260397:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777063.260400:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777063.260405:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777063.260409:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777063.260415:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777063.260422:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099828992, portal 25 00000100:00000001:0.0:1775777063.260424:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777063.260426:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751253504 : -114841958298112 : ffff978d45dece00) 00000100:00000040:0.0:1775777063.260431:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.260438:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777063.260441:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777063.260444:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777063.260447:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099828992, offset 0 00000400:00000200:0.0:1775777063.260453:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777063.260474:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.260480:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777063.260485:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777063.260489:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777063.260493:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777063.260496:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777063.260500:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4707:4707:256:4294967295] 192.168.201.139@tcp LPNI seq info [4707:4707:8:4294967295] 00000400:00000200:0.0:1775777063.260505:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.260516:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777063.260522:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.260527:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777063.260534:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777063.260538:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775777063.260543:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777063.260546:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775777063.260551:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.260560:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777063.260567:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777063.260571:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777063.260576:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777063.260578:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777063.260582:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777063.260586:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777063.260836:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777063.260841:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777063.260844:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775777063.260845:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777063.260847:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777063.260851:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777063.260853:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18778 00000400:00000200:3.0:1775777063.260858:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777063.260861:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777063.260864:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777063.260866:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260869:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.260871:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260873:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777063.260875:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777063.260876:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775777063.260878:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777063.260880:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775777063.260881:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775777063.260883:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775777063.260886:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260888:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.260890:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260891:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777063.260894:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775777063.260896:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777063.260898:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777063.260900:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777063.260905:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777063.260908:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.260909:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777063.260913:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777063.260916:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.260918:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777063.260923:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45defb80. 00000020:00000040:3.0:1775777063.260925:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:3.0:1775777063.260928:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777063.260929:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.260931:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777063.260932:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777063.260935:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777063.260938:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51159400. 00000100:00000001:3.0:1775777063.260942:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777063.260947:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777063.260956:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777063.260958:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777063.260962:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000930cefa] to pc [ptlrpcd_rcv+-1] req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.261755:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.261765:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.261774:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.261776:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777063.261799:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777063.261805:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777063.261808:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313452278 00000100:00000010:0.0:1775777063.261814:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777063.261820:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777063.261827:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313452278 00000100:00000010:0.0:1775777063.261830:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ba0. 00000100:00000001:0.0:1775777063.261833:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777063.261836:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777063.261839:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.261842:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777063.261845:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777063/real 1775777063] req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777063.261856:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777063.261858:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777063.261861:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777063.261863:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777063.261866:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777063.261869:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777063.261872:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777063.261881:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.261887:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777063.261889:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.261892:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.261893:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777063.261897:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.261904:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777063.261908:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.261915:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777063.261918:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.261920:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.261921:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.261923:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777063.261925:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777063.261928:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777063.261932:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777063.261935:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777063.261939:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777063.261946:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099828992:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777063.261951:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.261953:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.261956:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dece00 x1862033099828992/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.261962:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777063.261965:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777063.261985:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777063.261999:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777063.262001:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777063.262002:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777063.262004:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777063.262007:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159c00. 02000000:00000001:0.0:1775777063.262010:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777063.262012:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777063.262013:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dece00. 00000100:00000001:0.0:1775777063.262016:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777063.262017:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.262019:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.262026:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777063.262028:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.262031:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777063.262033:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777063.262034:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777063.262038:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.262045:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777063.262046:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.262049:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777063.262051:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777063.262054:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777063.262057:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829120:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777063.262061:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777063.262063:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777063.262066:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.262068:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777063.263403:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777063.263409:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777063.263419:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777063.263426:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777063.263432:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777063.263440:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099829120, portal 10 00000100:00000001:0.0:1775777063.263442:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777063.263444:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751265152 : -114841958286464 : ffff978d45defb80) 00000100:00000040:0.0:1775777063.263454:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.263462:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777063.263465:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777063.263467:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777063.263470:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099829120, offset 0 00000400:00000200:0.0:1775777063.263476:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777063.263484:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.263491:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777063.263496:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777063.263508:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777063.263512:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777063.263515:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777063.263519:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4708:4708:256:4294967295] 192.168.201.139@tcp LPNI seq info [4708:4708:8:4294967295] 00000400:00000200:0.0:1775777063.263524:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.263534:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777063.263540:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.263546:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777063.263552:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777063.263761:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775777063.263767:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777063.263771:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775777063.263776:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.263792:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777063.263798:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777063.263803:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777063.263808:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777063.263810:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777063.263813:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777063.263818:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.263829:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.263832:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.263840:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.263841:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777063.263846:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777063.263850:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777063.263853:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777063.263856:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313451015 00000100:00000001:0.0:1775777063.263859:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313452278 00000100:00000001:0.0:1775777063.263861:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313451015 00000100:00000001:0.0:1775777063.263863:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313452278 00000100:00000001:0.0:1775777063.263865:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777063.263868:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.263870:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777063.263874:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777063/real 1775777063] req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777063.263889:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777063.263891:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777063.263893:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777063.263895:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777063.263897:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777063.263899:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777063.263902:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777063.263910:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.263917:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777063.263918:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.263920:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.263922:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777063.263925:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.263932:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777063.263937:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.263944:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777063.263947:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.263948:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.263949:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.263952:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777063.263954:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777063.263957:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777063.263960:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777063.263963:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777063.264484:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777063.264632:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829120:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777063.264638:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.264640:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.264643:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45defb80 x1862033099829120/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.264650:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777063.264652:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777063.264656:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777063.264660:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777063.264663:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777063.264664:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777063.264667:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777063.264669:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51159400. 02000000:00000001:0.0:1775777063.264673:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777063.264675:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777063.264676:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45defb80. 00000100:00000001:0.0:1775777063.264680:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777063.264681:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.264683:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.264691:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777063.264693:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.264695:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777063.264697:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.267828:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777063.267839:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777063.267844:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775777063.267846:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777063.267850:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777063.267854:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777063.267862:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18778 00000400:00000200:3.0:1775777063.267872:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777063.267879:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777063.267882:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777063.267886:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.267889:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.267891:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.267894:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777063.267898:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777063.267900:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775777063.267903:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777063.267906:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775777063.267908:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775777063.267910:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775777063.267914:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.267916:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.267917:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.267919:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777063.267922:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775777063.267925:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777063.267927:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777063.267930:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777063.267938:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777063.267941:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.267943:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777063.267948:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777063.267951:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.267953:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777063.267964:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000020:00000040:3.0:1775777063.268818:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:3.0:1775777063.268826:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777063.268828:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.268831:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777063.268834:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777063.268838:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777063.268850:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115bc00. 00000100:00000001:3.0:1775777063.268859:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777063.268870:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777063.268881:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777063.268887:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777063.268891:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777063.268903:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777063.268907:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777063.268910:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775777063.268912:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777063.268913:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777063.268916:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777063.268920:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18778 00000400:00000200:3.0:1775777063.268926:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777063.268931:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777063.268933:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777063.268936:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.268938:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.268940:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.268942:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777063.268945:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777063.268946:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775777063.268948:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777063.268950:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775777063.268952:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775777063.268954:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775777063.268956:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.268958:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.268960:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.268961:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777063.268964:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:3.0:1775777063.268966:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777063.268968:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777063.268970:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777063.268974:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777063.268976:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.268978:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777063.268981:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777063.268984:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.268986:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777063.269002:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000020:00000040:3.0:1775777063.269005:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775777063.269007:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777063.269008:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.269010:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777063.269012:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777063.269013:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777063.269017:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a400. 00000100:00000001:3.0:1775777063.269020:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777063.269025:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777063.269033:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777063.269035:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777063.269038:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777063.269046:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777063.269049:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777063.269052:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1775777063.269053:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777063.269055:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777063.269058:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777063.269060:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18778 00000400:00000200:3.0:1775777063.269064:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777063.269067:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777063.269069:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777063.269082:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.269084:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.269086:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.269088:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777063.269097:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777063.269178:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775777063.269180:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777063.269182:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775777063.269184:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775777063.269186:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:3.0:1775777063.269189:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.269191:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777063.269193:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777063.269194:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777063.269196:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:3.0:1775777063.269199:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777063.269201:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777063.269203:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777063.269206:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777063.269208:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.269210:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777063.269212:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777063.269214:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.269215:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777063.269219:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775777063.269222:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:3.0:1775777063.269224:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777063.269225:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.269226:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777063.269228:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777063.269229:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777063.269233:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775777063.269236:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777063.269241:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777063.269248:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777063.269250:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777063.269253:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777063.269260:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777063.269266:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18788) 00000100:00000001:0.0:1775777063.269368:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777063.269371:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777063.269374:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777063.269379:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269387:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777063.269388:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.269391:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777063.269394:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777063.269396:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777063.269400:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829248:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777063.269403:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777063.269407:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777063.269410:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.269412:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777063.269417:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777063.269419:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777063.269424:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777063.269438:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777063.269444:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777063.269451:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099829248, portal 10 00000100:00000001:0.0:1775777063.269453:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777063.269454:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751262464 : -114841958289152 : ffff978d45def100) 00000100:00000040:0.0:1775777063.269466:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269473:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777063.269476:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777063.269479:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777063.269481:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099829248, offset 0 00000400:00000200:0.0:1775777063.269487:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777063.269495:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.269504:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777063.269508:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777063.269512:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777063.269516:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777063.269519:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777063.269523:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4709:4709:256:4294967295] 192.168.201.139@tcp LPNI seq info [4709:4709:8:4294967295] 00000400:00000200:0.0:1775777063.269527:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.269536:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777063.269542:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.269547:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777063.269561:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777063.269565:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775777063.269571:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777063.269574:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775777063.269590:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.269593:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777063.269599:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777063.269604:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777063.269609:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777063.269612:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777063.269615:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777063.269619:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269629:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.269632:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269639:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.269641:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777063.269646:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777063.269650:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777063.269653:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777063.269656:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313451022 00000100:00000001:0.0:1775777063.269659:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313452278 00000100:00000001:0.0:1775777063.269661:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313451022 00000100:00000001:0.0:1775777063.269663:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313452278 00000100:00000001:0.0:1775777063.269665:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777063.269667:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.269670:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777063.269673:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777063/real 1775777063] req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777063.269688:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777063.269690:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777063.269692:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777063.269694:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777063.269695:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777063.269698:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777063.269701:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777063.269710:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269716:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777063.269718:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.269720:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.269722:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777063.269725:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269731:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777063.269736:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269743:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777063.269745:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.269747:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.269748:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.269751:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777063.269753:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777063.269755:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777063.269759:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777063.269761:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777063.269765:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777063.269773:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829248:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777063.269802:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777063.269806:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269812:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777063.269814:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.269816:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777063.269818:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777063.269820:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777063.269823:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829376:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777063.269826:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777063.269828:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777063.269829:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.269831:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777063.269835:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775777063.269836:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777063.269839:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777063.269841:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777063.269844:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777063.269847:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099829376, portal 4 00000100:00000001:0.0:1775777063.269849:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777063.269850:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751261568 : -114841958290048 : ffff978d45deed80) 00000100:00000040:0.0:1775777063.269855:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269860:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777063.269862:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777063.269864:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777063.269866:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099829376, offset 0 00000400:00000200:0.0:1775777063.269870:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777063.269873:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.269877:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777063.269879:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777063.269882:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777063.269884:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777063.269886:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777063.269888:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4710:4710:256:4294967295] 192.168.201.139@tcp LPNI seq info [4710:4710:8:4294967295] 00000400:00000200:0.0:1775777063.269892:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.269897:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777063.269900:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.269903:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777063.269907:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777063.269909:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775777063.269912:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777063.269915:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775777063.269918:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.269920:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777063.269922:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777063.269925:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777063.269929:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777063.269930:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777063.269932:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777063.269935:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269942:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.269944:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.269950:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.269951:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777063.269954:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777063.269957:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777063.269959:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777063.269961:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313451032 00000100:00000001:0.0:1775777063.269964:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313452278 00000100:00000001:0.0:1775777063.269966:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313451032 00000100:00000001:0.0:1775777063.269968:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313452278 00000100:00000001:0.0:1775777063.269969:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777063.269971:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.269973:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777063.269976:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777063/real 1775777063] req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777063.269983:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777063.269984:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777063.269985:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777063.269999:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777063.270000:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777063.270003:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777063.270006:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777063.270013:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270018:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777063.270019:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270021:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.270022:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777063.270026:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270032:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777063.270035:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270041:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777063.270044:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270045:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.270047:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270049:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777063.270050:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777063.270052:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777063.270055:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777063.270057:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777063.270060:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777063.270067:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829376:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777063.270071:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777063.270074:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270080:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777063.270081:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.270083:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777063.270084:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777063.270086:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777063.270089:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829504:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777063.270092:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777063.270093:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777063.270095:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777063.270096:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777063.270109:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775777063.270111:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777063.270114:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777063.270117:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777063.270120:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777063.270122:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099829504, portal 4 00000100:00000001:0.0:1775777063.270125:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777063.270126:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751251712 : -114841958299904 : ffff978d45dec700) 00000100:00000040:0.0:1775777063.270131:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270137:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777063.270139:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777063.270141:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777063.270143:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099829504, offset 0 00000400:00000200:0.0:1775777063.270146:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777063.270150:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.270153:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777063.270156:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777063.270158:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777063.270160:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777063.270162:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777063.270165:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4711:4711:256:4294967295] 192.168.201.139@tcp LPNI seq info [4711:4711:8:4294967295] 00000400:00000200:0.0:1775777063.270169:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777063.270174:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777063.270178:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.270181:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777063.270185:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777063.270187:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775777063.270191:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777063.270194:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775777063.270197:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777063.270198:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777063.270201:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777063.270204:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777063.270206:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777063.270208:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777063.270210:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777063.270213:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270226:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.270229:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270234:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.270235:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777063.270238:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777063.270241:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777063.270243:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777063.270245:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313451023 00000100:00000001:0.0:1775777063.270247:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313452278 00000100:00000001:0.0:1775777063.270249:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313451023 00000100:00000001:0.0:1775777063.270251:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313452278 00000100:00000001:0.0:1775777063.270252:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777063.270254:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.270256:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777063.270259:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777063/real 1775777063] req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777063.270267:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777063.270268:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777063.270269:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777063.270271:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777063.270273:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777063.270275:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777063.270277:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777063.270285:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270290:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777063.270291:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270293:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.270294:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777063.270297:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270303:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777063.270307:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270312:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777063.270314:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270315:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777063.270316:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270318:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777063.270320:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777063.270322:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777063.270325:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777063.270327:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777063.270330:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777063.270336:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829504:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777063.270340:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270341:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.270344:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def100 x1862033099829248/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270350:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777063.270352:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777063.270355:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777063.270358:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777063.270360:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777063.270362:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777063.270364:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777063.270366:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115bc00. 02000000:00000001:0.0:1775777063.270369:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777063.270371:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777063.270372:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def100. 00000100:00000001:0.0:1775777063.270376:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777063.270377:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270379:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.270382:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deed80 x1862033099829376/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270387:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777063.270388:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777063.270389:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775777063.270391:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777063.270392:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777063.270393:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777063.270395:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777063.270396:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a400. 02000000:00000001:0.0:1775777063.270398:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777063.270399:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777063.270400:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deed80. 00000100:00000001:0.0:1775777063.270403:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777063.270404:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270405:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777063.270408:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec700 x1862033099829504/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777063.270414:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777063.270415:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777063.270416:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775777063.270418:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777063.270419:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777063.270420:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777063.270421:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777063.270422:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:0.0:1775777063.270424:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777063.270426:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777063.270427:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec700. 00000100:00000001:0.0:1775777063.270429:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777063.270430:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270432:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777063.270439:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777063.270441:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777063.270443:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777063.270445:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.455080:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777063.455088:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777063.455090:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777063.455097:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777063.455100:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777063.455105:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777063.455107:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777063.455109:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777063.455112:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777063.963922:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777063.963927:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777063.963930:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777063.963935:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777063.963939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777063.963943:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777063.963946:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777063.963948:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777063.963951:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777064.028066:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777064.028072:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777064.028087:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777064.028088:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777064.028092:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777064.028093:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777064.028096:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777064.028097:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777064.029776:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777064.029790:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777064.029795:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777064.477379:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777064.477385:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777064.477386:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777064.477392:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777064.477396:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777064.477399:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777064.477401:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777064.477403:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777064.477406:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777064.540695:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313452278, at 4313452288 00000100:00000010:1.0:1775777064.540712:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ba0. 00000100:00000040:0.0:1775777064.540803:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313452278, at 4313452289 00000100:00000010:0.0:1775777064.540810:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775777064.987929:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777064.987937:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777064.987939:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777064.987945:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777064.987949:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777064.987953:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777064.987955:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777064.987970:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777064.987973:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777065.051933:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777065.051939:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777065.051945:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777065.051947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777065.051950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777065.051951:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777065.051954:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777065.051988:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777065.052582:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777065.052585:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777065.052588:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777065.501541:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777065.501547:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777065.501549:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777065.501554:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777065.501558:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777065.501561:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777065.501564:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777065.501567:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777065.501570:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777066.011881:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777066.011888:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777066.011890:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777066.011897:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777066.011901:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777066.011905:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777066.011907:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777066.011911:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777066.011914:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777066.076742:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777066.076748:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777066.076754:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777066.076755:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777066.076759:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777066.076760:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777066.076764:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777066.076765:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777066.077081:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777066.077084:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777066.077088:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777066.523964:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777066.523971:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777066.523974:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777066.523980:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777066.523984:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777066.523988:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777066.523991:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777066.523993:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777066.523996:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777067.035874:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777067.035880:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777067.035883:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777067.035888:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777067.035892:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777067.035896:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777067.035898:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777067.035900:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777067.035903:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777067.099918:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777067.099934:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777067.099940:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777067.099941:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777067.099944:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777067.099946:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777067.099949:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777067.099950:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777067.100057:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777067.100061:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777067.100064:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777067.548269:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777067.548277:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777067.548279:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777067.548285:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777067.548289:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777067.548293:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777067.548296:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777067.548298:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777067.548301:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777068.060271:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777068.060279:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777068.060281:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777068.060286:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777068.060290:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777068.060295:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777068.060297:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777068.060300:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777068.060302:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777068.124471:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777068.124478:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777068.124485:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777068.124486:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777068.124489:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777068.124490:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777068.124493:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777068.124494:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777068.124611:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777068.124615:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777068.124618:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775777068.386145:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777068.386155:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775777068.386158:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777068.386161:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777068.386166:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777068.386173:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18783 00000400:00000200:0.0:1775777068.386183:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.386189:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777068.386192:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777068.386196:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386199:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386201:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386204:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777068.386208:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777068.386210:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775777068.386213:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777068.386216:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775777068.386218:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775777068.386221:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775777068.386223:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386225:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386227:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386229:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777068.386231:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775777068.386234:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386236:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.386239:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777068.386243:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.386245:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386248:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775777068.386251:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777068.386260:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775777068.386264:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775777068.386268:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777068.386271:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386285:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777068.386288:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777068.386291:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777068.386299:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c000. 00000100:00000001:0.0:1775777068.386306:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.386314:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777068.386324:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777068.386328:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.386333:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.386361:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777068.386364:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777068.386368:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775777068.386369:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777068.386370:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777068.386373:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777068.386375:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18783 00000400:00000200:0.0:1775777068.386379:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.386381:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777068.386383:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777068.386386:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386388:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386390:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386392:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777068.386394:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777068.386395:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775777068.386397:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777068.386398:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775777068.386400:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775777068.386402:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775777068.386405:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386407:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386409:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386410:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777068.386412:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775777068.386414:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386416:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.386418:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777068.386423:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.386425:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386427:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777068.386430:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775777068.386433:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386435:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777068.386439:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775777068.386441:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775777068.386444:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777068.386445:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386447:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777068.386448:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777068.386450:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777068.386454:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759ec00. 00000100:00000001:0.0:1775777068.386457:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.386461:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777068.386468:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777068.386470:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.386473:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.386481:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777068.386483:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777068.386486:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775777068.386488:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777068.386489:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777068.386492:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777068.386494:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18783 00000400:00000200:0.0:1775777068.386498:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.386501:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777068.386502:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777068.386505:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386507:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386509:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386512:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777068.386515:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777068.386516:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775777068.386517:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777068.386519:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775777068.386521:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775777068.386522:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775777068.386525:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386526:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386527:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386529:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777068.386531:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775777068.386533:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386536:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.386537:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777068.386540:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.386541:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386543:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777068.386545:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775777068.386546:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386548:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777068.386552:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775777068.386554:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775777068.386556:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777068.386557:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386558:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777068.386559:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777068.386561:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777068.386564:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775777068.386566:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.386570:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777068.386582:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777068.386584:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.386587:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.386594:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777068.386597:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777068.386599:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775777068.386601:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777068.386602:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777068.386604:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777068.386606:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18783 00000400:00000200:0.0:1775777068.386610:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.386612:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777068.386614:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777068.386616:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386618:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386620:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386622:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777068.386623:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777068.386624:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775777068.386626:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777068.386628:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775777068.386629:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775777068.386631:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775777068.386634:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386635:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386637:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386639:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777068.386641:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775777068.386643:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386645:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.386646:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777068.386648:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.386650:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386651:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777068.386653:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775777068.386655:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386657:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777068.386660:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775777068.386662:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775777068.386664:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777068.386665:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386667:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777068.386668:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777068.386669:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777068.386672:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775777068.386674:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.386678:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777068.386685:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777068.386687:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.386690:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.386697:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777068.386699:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777068.386701:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775777068.386703:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777068.386704:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777068.386706:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777068.386709:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18783 00000400:00000200:0.0:1775777068.386712:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.386715:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777068.386717:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777068.386719:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386721:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386723:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386725:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777068.386727:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777068.386728:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775777068.386729:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777068.386731:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775777068.386733:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775777068.386735:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775777068.386738:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386740:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777068.386741:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386743:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777068.386745:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775777068.386748:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777068.386750:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.386751:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777068.386754:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.386755:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386757:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777068.386759:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775777068.386761:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386763:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777068.386766:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775777068.386768:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775777068.386770:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777068.386772:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.386773:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777068.386775:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777068.386776:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777068.386844:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775777068.386848:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.386853:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777068.386860:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777068.386862:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.386865:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.386872:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.386877:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18793) 00000100:00000001:3.0:1775777068.386911:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775777068.386915:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777068.386917:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775777068.386923:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777068.386932:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777068.386934:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777068.386937:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775777068.386940:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777068.386943:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777068.386946:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099829632:192.168.201.139@tcp:250:kworker.0 00000100:00000001:3.0:1775777068.386950:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775777068.386953:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775777068.386955:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777068.386958:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777068.387339:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775777068.387343:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777068.387349:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777068.387354:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777068.387360:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777068.387367:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099829632, portal 25 00000100:00000001:0.0:1775777068.387369:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777068.387370:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757056 : -114841967794560 : ffff978d454de680) 00000100:00000040:0.0:1775777068.387376:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.387384:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777068.387397:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777068.387399:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777068.387401:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099829632, offset 0 00000400:00000200:0.0:1775777068.387407:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777068.387414:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.387420:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777068.387424:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777068.387429:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777068.387432:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777068.387434:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777068.387438:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4712:4712:256:4294967295] 192.168.201.139@tcp LPNI seq info [4712:4712:8:4294967295] 00000400:00000200:0.0:1775777068.387442:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.387450:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777068.387455:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.387459:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777068.387465:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777068.387468:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775777068.387473:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777068.387476:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775777068.387479:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.387482:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777068.387488:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777068.387492:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777068.387496:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777068.387498:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777068.387502:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777068.387505:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.387514:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.387516:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.387522:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.387524:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777068.387529:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777068.387544:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777068.387546:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313457404 00000100:00000010:0.0:1775777068.387551:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777068.387557:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777068.387561:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313457404 00000100:00000010:0.0:1775777068.387565:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237780. 00000100:00000001:0.0:1775777068.387567:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777068.387570:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777068.387572:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.387575:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777068.387579:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777068/real 1775777068] req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777068.387587:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777068.387589:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777068.387591:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777068.387593:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777068.387595:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777068.387597:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777068.387600:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777068.387608:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.387615:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777068.387616:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.387619:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.387621:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777068.387624:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.387629:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777068.387632:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.387638:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777068.387640:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.387641:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.387642:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.387644:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777068.387646:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777068.387648:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777068.387652:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777068.387956:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777068.387963:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777068.387970:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099829632:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777068.387974:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777068.387977:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.387982:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777068.387984:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.387986:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777068.387987:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.387989:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.387992:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829760:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777068.387994:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777068.387996:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777068.388260:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.388263:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777068.388267:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775777068.388270:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777068.388274:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777068.388277:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777068.388281:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777068.388284:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099829760, portal 10 00000100:00000001:0.0:1775777068.388286:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777068.388288:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741762432 : -114841967789184 : ffff978d454dfb80) 00000100:00000040:0.0:1775777068.388295:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.388302:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777068.388304:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777068.388306:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777068.388309:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099829760, offset 0 00000400:00000200:0.0:1775777068.388313:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777068.388317:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.388321:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777068.388324:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777068.388326:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777068.388329:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777068.388331:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777068.388334:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4713:4713:256:4294967295] 192.168.201.139@tcp LPNI seq info [4713:4713:8:4294967295] 00000400:00000200:0.0:1775777068.388337:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.388344:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777068.388349:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.388352:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777068.388356:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777068.388359:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775777068.388363:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777068.388365:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775777068.388369:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.388370:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777068.388374:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777068.388377:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777068.388379:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777068.388381:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777068.388384:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777068.388398:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.388406:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.388409:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.388420:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.388421:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777068.388425:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.388428:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777068.388431:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777068.388433:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313456140 00000100:00000001:0.0:1775777068.388436:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313457404 00000100:00000001:0.0:1775777068.388438:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313456140 00000100:00000001:0.0:1775777068.388439:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313457404 00000100:00000001:0.0:1775777068.388441:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777068.388443:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.388445:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777068.388448:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777068/real 1775777068] req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777068.388456:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777068.388457:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777068.388459:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777068.388461:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777068.388462:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777068.388464:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777068.388467:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777068.388474:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.388480:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777068.388481:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.388483:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.388485:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777068.388488:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.388495:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777068.388499:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.388505:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777068.388507:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.388508:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.388510:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.388512:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777068.388514:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777068.388516:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777068.388519:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777068.388521:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777068.388524:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777068.388541:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829760:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777068.388545:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777068.388549:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.388554:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777068.388555:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.388558:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777068.388560:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.388561:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.388564:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829888:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777068.388567:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777068.388570:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777068.388572:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.388573:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777068.388577:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759d000. 02000000:00000001:0.0:1775777068.388578:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777068.388581:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777068.388583:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777068.388586:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777068.388589:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099829888, portal 10 00000100:00000001:0.0:1775777068.388591:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777068.388592:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:0.0:1775777068.388597:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.388602:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777068.388604:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777068.388605:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777068.388607:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099829888, offset 0 00000400:00000200:0.0:1775777068.388611:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777068.388614:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.388617:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777068.388620:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777068.388622:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777068.388624:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777068.388626:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777068.388629:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4714:4714:256:4294967295] 192.168.201.139@tcp LPNI seq info [4714:4714:8:4294967295] 00000400:00000200:0.0:1775777068.388632:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.388637:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777068.388640:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.388643:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777068.388646:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777068.388648:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775777068.388651:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777068.388978:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775777068.388985:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.388987:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777068.388991:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777068.388994:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777068.388997:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777068.389645:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777068.389651:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777068.389728:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.389738:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.389741:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.389747:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.389748:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777068.389754:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.389884:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777068.389890:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777068.389892:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313456143 00000100:00000001:0.0:1775777068.389895:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313457404 00000100:00000001:0.0:1775777068.389907:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313456143 00000100:00000001:0.0:1775777068.389909:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313457404 00000100:00000001:0.0:1775777068.389911:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777068.389913:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.389915:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777068.389920:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777068/real 1775777068] req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777068.389929:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777068.389930:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777068.389933:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777068.389936:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777068.389937:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777068.389940:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777068.389943:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777068.389951:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.389958:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777068.389959:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.389961:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.389963:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777068.389966:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.389973:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777068.389976:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.389983:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777068.389985:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.389987:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.389988:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.389990:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777068.389992:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777068.389994:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777068.390119:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777068.390123:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777068.390127:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777068.390134:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099829888:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777068.390624:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777068.390630:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.390637:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777068.390639:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.390642:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777068.390645:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.390647:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.390650:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830016:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777068.390729:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777068.390734:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777068.390736:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.390739:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777068.390747:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775777068.390750:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777068.390754:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777068.390935:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777068.390941:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777068.390947:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099830016, portal 4 00000100:00000001:0.0:1775777068.390949:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777068.390950:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:0.0:1775777068.390958:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.390964:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777068.390967:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777068.390969:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777068.390971:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099830016, offset 0 00000400:00000200:0.0:1775777068.390976:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777068.390981:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.390988:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777068.390991:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777068.390995:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777068.391473:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777068.391477:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777068.391481:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4715:4715:256:4294967295] 192.168.201.139@tcp LPNI seq info [4715:4715:8:4294967295] 00000400:00000200:0.0:1775777068.391486:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.391494:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777068.391500:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.391506:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777068.391512:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777068.391515:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775777068.391520:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777068.391523:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775777068.391528:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.391620:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777068.391627:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777068.391631:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777068.391636:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777068.391638:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777068.391642:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777068.391646:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.392218:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.392225:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.392233:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.392235:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777068.392242:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.392247:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777068.392250:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777068.392253:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313456154 00000100:00000001:0.0:1775777068.392256:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313457404 00000100:00000001:0.0:1775777068.392258:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313456154 00000100:00000001:0.0:1775777068.392260:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313457404 00000100:00000001:0.0:1775777068.392263:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777068.392266:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.392268:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777068.392272:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777068/real 1775777068] req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777068.392282:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777068.392283:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777068.392285:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777068.392288:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777068.392289:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777068.392291:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777068.392294:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777068.392303:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.392309:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777068.392311:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.392313:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.392315:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777068.392318:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.392325:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777068.392329:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.392335:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777068.392338:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.392339:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.392340:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.392343:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777068.392345:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777068.392348:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777068.392353:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777068.392355:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777068.392359:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777068.392366:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830016:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777068.392370:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777068.392374:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.392380:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777068.392381:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.392384:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777068.395508:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777068.395516:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777068.395529:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830144:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777068.395881:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777068.395886:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777068.395889:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777068.395892:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777068.395911:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775777068.395914:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777068.395922:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777068.395927:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777068.395934:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777068.395941:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099830144, portal 4 00000100:00000001:0.0:1775777068.395944:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777068.395945:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:0.0:1775777068.395955:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.395963:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777068.395967:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777068.395969:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777068.395972:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099830144, offset 0 00000400:00000200:0.0:1775777068.395979:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777068.395987:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.395995:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777068.396742:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777068.396748:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777068.396751:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777068.396754:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777068.396978:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4716:4716:256:4294967295] 192.168.201.139@tcp LPNI seq info [4716:4716:8:4294967295] 00000400:00000200:0.0:1775777068.396985:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777068.396994:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777068.399995:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.400008:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777068.400016:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777068.400019:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775777068.400025:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777068.400028:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775777068.400032:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777068.400034:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777068.400042:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777068.400047:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777068.400056:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777068.400059:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777068.400064:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777068.400073:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400085:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.400088:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400095:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.400097:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777068.400104:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777068.400109:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777068.400116:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777068.400119:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313456153 00000100:00000001:0.0:1775777068.400122:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313457404 00000100:00000001:0.0:1775777068.400125:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313456153 00000100:00000001:0.0:1775777068.400127:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313457404 00000100:00000001:0.0:1775777068.400129:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777068.400132:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.400135:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777068.400140:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777068/real 1775777068] req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777068.400150:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777068.400152:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777068.400155:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777068.400158:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777068.400159:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777068.400161:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777068.400165:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777068.400173:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400181:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777068.400183:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400185:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.400187:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777068.400191:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400198:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777068.400202:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400211:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777068.400214:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400216:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777068.400217:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400220:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777068.400222:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777068.400225:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777068.400230:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777068.400233:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777068.400237:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777068.400245:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830144:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777068.400250:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400252:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.400256:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de680 x1862033099829632/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400265:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777068.400267:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777068.400271:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d800. 02000000:00000001:0.0:1775777068.400274:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777068.400277:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777068.400278:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777068.400280:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777068.400282:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c000. 02000000:00000001:0.0:1775777068.400285:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777068.400287:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777068.400289:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de680. 00000100:00000001:0.0:1775777068.400292:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777068.400293:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400295:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.400299:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dfb80 x1862033099829760/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400305:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777068.400307:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777068.400308:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d400. 02000000:00000001:0.0:1775777068.400310:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777068.400311:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777068.400312:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777068.400314:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777068.400315:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759ec00. 02000000:00000001:0.0:1775777068.400317:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777068.400318:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777068.400319:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dfb80. 00000100:00000001:0.0:1775777068.400322:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777068.400323:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400324:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.400328:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099829888/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400334:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777068.400335:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777068.400337:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759d000. 02000000:00000001:0.0:1775777068.400339:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777068.400340:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777068.400341:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777068.400343:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777068.400344:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777068.400346:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777068.400348:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777068.400349:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:0.0:1775777068.400351:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777068.400352:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400354:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.400358:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099830016/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400370:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777068.400371:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777068.400373:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775777068.400375:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777068.400376:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777068.400377:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777068.400379:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777068.400380:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777068.400383:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777068.400384:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777068.400385:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:0.0:1775777068.400387:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777068.400388:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400390:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777068.400393:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099830144/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777068.400400:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777068.400401:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777068.400402:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775777068.400404:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777068.400405:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777068.400406:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777068.400407:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777068.400409:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777068.400410:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777068.400411:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777068.400413:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:0.0:1775777068.400414:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777068.400416:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400418:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777068.400425:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777068.400427:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777068.400430:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777068.400431:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777068.577331:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777068.577339:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777068.577341:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777068.577346:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777068.577350:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777068.577354:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777068.577356:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777068.577358:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777068.577361:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777069.083920:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777069.083927:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777069.083929:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777069.083935:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777069.083939:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777069.083943:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777069.083945:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777069.083948:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777069.083951:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777069.148035:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777069.148043:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777069.148046:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777069.148050:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777069.148054:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777069.148057:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777069.148059:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777069.148061:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777069.148065:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777069.148067:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777069.148070:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777069.148072:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777069.148075:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777069.148076:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777069.148173:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777069.148176:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777069.148179:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777069.596221:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777069.596227:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777069.596229:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777069.596236:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777069.596240:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777069.596243:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777069.596245:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777069.596248:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777069.596251:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777069.659907:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313457404, at 4313457408 00000100:00000040:1.0:1775777069.659907:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313457404, at 4313457408 00000100:00000010:1.0:1775777069.659922:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237780. 00000100:00000010:0.0:1775777069.659923:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775777070.109065:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777070.109072:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777070.109074:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777070.109080:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777070.109084:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777070.109087:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777070.109090:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777070.109092:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777070.109095:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777070.172745:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777070.172751:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777070.172757:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777070.172759:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777070.172763:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777070.172764:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777070.172768:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777070.172769:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777070.172942:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777070.172945:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777070.172949:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777070.624404:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777070.624412:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777070.624414:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777070.624422:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777070.624426:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777070.624429:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777070.624432:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777070.624435:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777070.624438:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777071.132710:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777071.132717:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777071.132719:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777071.132724:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777071.132728:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777071.132732:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777071.132734:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777071.132736:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777071.132739:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777071.196018:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777071.196025:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777071.196031:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777071.196032:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777071.196035:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777071.196036:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777071.196040:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777071.196041:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777071.196142:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777071.196146:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777071.196149:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777071.643967:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777071.643974:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777071.643976:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777071.643981:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777071.643984:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777071.643988:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777071.643990:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777071.643992:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777071.643995:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777072.156031:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777072.156038:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777072.156040:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777072.156045:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777072.156049:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777072.156052:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777072.156055:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777072.156057:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777072.156060:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219947:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777072.219953:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219958:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777072.219962:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777072.219964:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777072.219968:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219971:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777072.219973:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219975:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219977:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219981:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777072.219983:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777072.219984:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777072.219985:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219987:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777072.219988:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219989:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219991:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219995:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777072.219996:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777072.219997:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777072.219998:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.219999:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777072.220000:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.220002:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777072.220003:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777072.220119:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777072.220123:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777072.220126:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777072.670036:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777072.670044:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777072.670046:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777072.670053:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777072.670057:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777072.670061:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777072.670064:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777072.670066:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777072.670070:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777073.179974:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777073.179982:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777073.179984:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777073.179991:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777073.179994:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777073.179999:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777073.180002:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777073.180004:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777073.180007:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777073.243987:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777073.243994:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777073.244001:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777073.244002:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777073.244006:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777073.244007:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777073.244010:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777073.244011:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777073.244595:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777073.244599:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777073.244602:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775777073.500195:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777073.500205:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775777073.500208:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777073.500212:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777073.500217:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777073.500225:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18788 00000400:00000200:2.0:1775777073.500235:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777073.500540:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777073.500543:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777073.500546:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.500549:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.500552:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.500554:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777073.500558:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777073.500560:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:2.0:1775777073.500563:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777073.500565:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:2.0:1775777073.500568:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:2.0:1775777073.500570:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:2.0:1775777073.500573:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.500575:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.500577:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.500578:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777073.500581:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:2.0:1775777073.500584:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777073.500586:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777073.500589:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777073.500592:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777073.500595:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.500597:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775777073.500599:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777073.500610:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7e680. 00000020:00000040:2.0:1775777073.500614:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:2.0:1775777073.500618:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777073.500619:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.500622:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777073.500625:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777073.500629:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777073.500637:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506400. 00000100:00000001:2.0:1775777073.500644:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777073.500654:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777073.500665:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777073.500669:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777073.500674:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000086433f13] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.500817:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777073.500822:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777073.500824:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777073.500830:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.500838:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777073.500839:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.500843:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777073.500846:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777073.500849:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777073.500853:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099830272:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777073.500856:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777073.500861:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777073.500863:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.500866:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777073.500871:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777073.500874:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777073.500880:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777073.500883:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777073.500890:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777073.500898:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099830272, portal 25 00000100:00000001:0.0:1775777073.500900:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777073.500902:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288720512 : -114841420831104 : ffff978d65e7e680) 00000100:00000040:0.0:1775777073.500907:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.500922:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777073.500925:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777073.500927:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777073.500930:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099830272, offset 0 00000400:00000200:0.0:1775777073.500937:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777073.500944:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.500950:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777073.500955:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777073.500959:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777073.500962:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777073.500964:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777073.500968:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4717:4717:256:4294967295] 192.168.201.139@tcp LPNI seq info [4717:4717:8:4294967295] 00000400:00000200:0.0:1775777073.500973:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.500982:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777073.500987:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.500992:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777073.500998:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777073.501001:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775777073.501006:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777073.501009:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775777073.501014:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.501016:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777073.501022:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777073.501026:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777073.501032:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777073.501034:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777073.501039:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777073.501042:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777073.501224:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777073.501229:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777073.501233:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775777073.501234:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777073.501245:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777073.501249:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777073.501253:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18788 00000400:00000200:2.0:1775777073.501258:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777073.501260:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777073.501262:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777073.501265:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.501267:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.501268:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.501270:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777073.501272:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777073.501274:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:2.0:1775777073.501275:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777073.501277:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:2.0:1775777073.501279:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:2.0:1775777073.501281:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:2.0:1775777073.501284:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.501286:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.501287:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.501289:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777073.501292:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:2.0:1775777073.501294:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777073.501296:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777073.501298:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777073.501305:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777073.501307:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.501309:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777073.501312:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777073.501315:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.501317:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777073.501320:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ea00. 00000020:00000040:2.0:1775777073.501323:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:2.0:1775777073.501326:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777073.501327:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.501329:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777073.501330:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777073.501333:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777073.501337:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51505c00. 00000100:00000001:2.0:1775777073.501341:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777073.501346:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777073.501354:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777073.501356:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777073.501360:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fc493d27] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.501557:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.501563:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.501571:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.501572:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777073.501581:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777073.501585:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777073.501588:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313462518 00000100:00000010:0.0:1775777073.501594:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777073.501602:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777073.501610:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313462518 00000100:00000010:0.0:1775777073.501612:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237d80. 00000100:00000001:0.0:1775777073.501615:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777073.501618:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777073.501621:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.501622:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777073.501627:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777073/real 1775777073] req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777073.501637:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777073.501638:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777073.501642:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777073.501644:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777073.501646:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777073.501648:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777073.501650:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777073.501658:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.501665:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777073.501667:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.501669:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.501671:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777073.501674:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.501681:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777073.501685:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.501692:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777073.502061:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502063:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.502065:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502067:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777073.502069:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777073.502072:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777073.502077:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777073.502079:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777073.502084:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777073.502092:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099830272:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777073.502097:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502099:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.502101:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7e680 x1862033099830272/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502108:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777073.502110:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777073.502113:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777073.502117:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777073.502119:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777073.502120:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777073.502122:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777073.502124:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506400. 02000000:00000001:0.0:1775777073.502129:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777073.502131:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777073.502133:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7e680. 00000100:00000001:0.0:1775777073.502136:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777073.502137:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502139:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502148:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777073.502150:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.502152:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777073.502154:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777073.502155:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777073.502159:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502165:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777073.502167:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.502169:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777073.502171:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777073.502173:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777073.502175:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830400:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777073.502178:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777073.502181:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777073.502182:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.502184:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777073.502187:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777073.502189:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777073.502194:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777073.502197:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777073.502202:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777073.502207:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099830400, portal 10 00000100:00000001:0.0:1775777073.502209:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777073.502210:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288721408 : -114841420830208 : ffff978d65e7ea00) 00000100:00000040:0.0:1775777073.502217:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502224:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777073.502226:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777073.502228:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777073.502231:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099830400, offset 0 00000400:00000200:0.0:1775777073.502235:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777073.502307:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.502313:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777073.502317:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777073.502321:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777073.502324:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777073.502327:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777073.502329:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4718:4718:256:4294967295] 192.168.201.139@tcp LPNI seq info [4718:4718:8:4294967295] 00000400:00000200:0.0:1775777073.502333:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.502341:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777073.502347:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.502351:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777073.502357:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777073.502360:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775777073.502365:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777073.502368:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775777073.502372:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.502374:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777073.502379:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777073.502383:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777073.502387:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777073.502388:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777073.502391:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777073.502394:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502402:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.502405:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502411:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.502413:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777073.502417:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777073.502421:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777073.502424:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777073.502427:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313461253 00000100:00000001:0.0:1775777073.502429:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313462518 00000100:00000001:0.0:1775777073.502431:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313461253 00000100:00000001:0.0:1775777073.502433:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313462518 00000100:00000001:0.0:1775777073.502435:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777073.502438:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.502440:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777073.502444:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777073/real 1775777073] req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777073.502452:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777073.502453:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777073.502455:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777073.502457:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777073.502458:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777073.502460:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777073.502463:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777073.502470:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502476:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777073.502478:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502479:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.502480:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777073.502483:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502489:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777073.502492:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502500:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777073.502502:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502504:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.502504:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502506:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777073.502508:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777073.502511:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777073.502514:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777073.502516:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777073.502519:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777073.502525:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830400:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777073.502528:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502530:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.502532:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ea00 x1862033099830400/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.502537:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777073.502539:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777073.502540:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777073.502543:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777073.502544:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777073.502545:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777073.502547:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777073.502548:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51505c00. 02000000:00000001:0.0:1775777073.502550:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777073.502551:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777073.502552:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ea00. 00000100:00000001:0.0:1775777073.502555:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777073.502556:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502557:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.502560:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777073.502562:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.502564:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777073.502565:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.503094:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777073.503097:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.503147:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777073.503153:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777073.503157:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775777073.503159:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777073.503162:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777073.503166:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777073.503171:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18788 00000400:00000200:2.0:1775777073.503177:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777073.503181:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777073.503184:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777073.503187:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503190:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.503192:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503194:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777073.503197:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777073.503199:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:2.0:1775777073.503201:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777073.503204:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:2.0:1775777073.503206:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:2.0:1775777073.503208:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:2.0:1775777073.503210:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503212:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.503213:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503215:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777073.503218:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:2.0:1775777073.503221:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503223:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777073.503225:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777073.503230:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777073.503233:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.503235:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777073.503438:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775777073.503441:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.503443:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777073.503450:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ca80. 00000020:00000040:2.0:1775777073.503454:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:2.0:1775777073.503457:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777073.503459:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.503461:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777073.503463:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777073.503467:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777073.503474:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506c00. 00000100:00000001:2.0:1775777073.503479:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777073.503486:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777073.503495:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777073.503499:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777073.503503:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a559b88] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777073.503523:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777073.503526:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777073.503530:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775777073.503531:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777073.503532:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777073.503535:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777073.503537:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18788 00000400:00000200:2.0:1775777073.503542:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777073.503544:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777073.503546:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777073.503548:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503550:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.503552:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503554:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777073.503556:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777073.503557:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:2.0:1775777073.503558:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777073.503560:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:2.0:1775777073.503562:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:2.0:1775777073.503563:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:2.0:1775777073.503566:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503567:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.503569:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503570:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777073.503572:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:2.0:1775777073.503575:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503576:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777073.503578:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777073.503581:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777073.503583:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.503585:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777073.503587:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777073.503589:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.503590:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777073.503593:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7d180. 00000020:00000040:2.0:1775777073.503595:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:2.0:1775777073.503597:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777073.503598:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.503599:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777073.503600:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777073.503602:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777073.503605:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51507c00. 00000100:00000001:2.0:1775777073.503607:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777073.503611:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777073.503622:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777073.503624:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777073.503627:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008292136e] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777073.503634:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775777073.503636:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775777073.503639:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775777073.503640:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775777073.503641:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775777073.503644:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775777073.503646:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18788 00000400:00000200:2.0:1775777073.503649:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777073.503652:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:2.0:1775777073.503654:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775777073.503656:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503658:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.503660:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503661:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:2.0:1775777073.503663:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775777073.503664:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:2.0:1775777073.503665:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775777073.503667:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:2.0:1775777073.503669:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:2.0:1775777073.503670:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:2.0:1775777073.503673:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503675:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775777073.503677:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503678:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:2.0:1775777073.503681:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:2.0:1775777073.503684:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:2.0:1775777073.503686:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777073.503688:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775777073.503690:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777073.503692:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.503864:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777073.503866:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777073.503868:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777073.503873:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.503879:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777073.503881:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.503883:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777073.503885:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777073.503887:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777073.503889:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830528:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777073.503892:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777073.503895:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777073.503897:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.503899:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777073.503903:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777073.503904:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777073.503909:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777073.503912:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777073.503991:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777073.503999:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099830528, portal 10 00000100:00000001:0.0:1775777073.504001:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777073.504002:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288713344 : -114841420838272 : ffff978d65e7ca80) 00000100:00000040:0.0:1775777073.504010:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.504017:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777073.504020:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777073.504023:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777073.504026:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099830528, offset 0 00000400:00000200:0.0:1775777073.504032:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777073.504039:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.504045:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777073.504050:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777073.504053:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777073.504057:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777073.504059:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777073.504063:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4719:4719:256:4294967295] 192.168.201.139@tcp LPNI seq info [4719:4719:8:4294967295] 00000400:00000200:0.0:1775777073.504068:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.504076:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777073.504082:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.504086:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777073.504092:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777073.504095:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775777073.504100:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777073.504103:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775777073.504107:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.504109:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777073.504114:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777073.504118:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777073.504123:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777073.504125:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777073.504128:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777073.504132:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.504141:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.504145:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.504151:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.504153:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777073.504157:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777073.504162:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777073.504165:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777073.504167:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313461257 00000100:00000001:0.0:1775777073.504170:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313462518 00000100:00000001:0.0:1775777073.504173:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313461257 00000100:00000001:0.0:1775777073.504175:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313462518 00000100:00000001:0.0:1775777073.504177:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777073.504180:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.504182:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777073.504186:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777073/real 1775777073] req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777073.504195:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777073.504196:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777073.504198:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777073.504201:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777073.504202:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777073.504204:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777073.504207:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777073.504216:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.504222:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777073.504224:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.504226:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.504228:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777073.504231:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777073.504373:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775777073.504375:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1775777073.504378:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.504380:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775777073.504384:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65e7ce00. 00000020:00000040:2.0:1775777073.504387:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:2.0:1775777073.504390:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775777073.504391:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777073.504393:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775777073.504394:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775777073.504396:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775777073.504400:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51506800. 00000100:00000001:2.0:1775777073.504403:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777073.504408:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775777073.504416:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775777073.504418:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777073.504421:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050a5f9f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777073.504429:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777073.504434:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18798) 00000100:00000001:0.0:1775777073.504543:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777073.504548:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.504554:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777073.504557:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.504558:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.504560:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.504562:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777073.504564:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777073.504567:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777073.504570:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777073.504572:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777073.504575:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777073.504582:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830528:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777073.504586:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777073.504589:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.504594:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777073.504595:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.504598:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777073.504599:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777073.504601:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777073.504603:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830656:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777073.504606:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777073.504607:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777073.504609:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.504611:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777073.504614:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775777073.504615:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777073.504619:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777073.504621:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777073.504625:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777073.504628:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099830656, portal 4 00000100:00000001:0.0:1775777073.504630:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777073.504631:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288715136 : -114841420836480 : ffff978d65e7d180) 00000100:00000040:0.0:1775777073.504635:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.504645:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777073.504647:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777073.504649:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777073.504651:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099830656, offset 0 00000400:00000200:0.0:1775777073.504654:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777073.504658:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.504661:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777073.504664:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777073.504666:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777073.504668:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777073.504669:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777073.504672:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4720:4720:256:4294967295] 192.168.201.139@tcp LPNI seq info [4720:4720:8:4294967295] 00000400:00000200:0.0:1775777073.504675:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.504680:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777073.504684:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.504686:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777073.504690:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777073.504692:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775777073.504867:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777073.504871:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775777073.504878:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.504880:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777073.504884:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777073.504888:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777073.504891:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777073.504893:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777073.504895:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777073.504899:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.504906:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.504909:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505001:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.505002:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777073.505007:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777073.505010:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777073.505014:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777073.505015:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313461267 00000100:00000001:0.0:1775777073.505018:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313462518 00000100:00000001:0.0:1775777073.505020:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313461267 00000100:00000001:0.0:1775777073.505021:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313462518 00000100:00000001:0.0:1775777073.505024:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777073.505026:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.505028:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777073.505032:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777073/real 1775777073] req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777073.505040:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777073.505041:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777073.505043:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777073.505046:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777073.505047:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777073.505049:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777073.505052:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777073.505060:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505066:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777073.505068:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505069:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.505070:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777073.505074:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505081:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777073.505084:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505090:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777073.505092:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505093:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.505094:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505096:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777073.505097:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777073.505100:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777073.505103:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777073.505105:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777073.505109:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777073.505115:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830656:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777073.505119:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505120:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.505123:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ca80 x1862033099830528/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505130:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777073.505132:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777073.505135:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777073.505138:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777073.505139:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777073.505141:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777073.505143:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777073.505146:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506c00. 02000000:00000001:0.0:1775777073.505149:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777073.505151:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777073.505153:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ca80. 00000100:00000001:0.0:1775777073.505156:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777073.505157:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505158:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.505161:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7d180 x1862033099830656/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505167:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777073.505168:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777073.505170:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775777073.505172:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777073.505173:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777073.505174:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777073.505176:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777073.505177:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51507c00. 02000000:00000001:0.0:1775777073.505180:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777073.505181:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777073.505182:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7d180. 00000100:00000001:0.0:1775777073.505184:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777073.505186:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505187:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505194:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777073.505195:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.505198:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777073.505199:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777073.505201:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777073.505204:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505211:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777073.505212:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.505214:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777073.505216:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777073.505218:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777073.505221:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830784:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777073.505227:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777073.505229:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777073.505230:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777073.505232:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777073.505236:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775777073.505305:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777073.505308:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777073.505311:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777073.505315:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777073.505319:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099830784, portal 4 00000100:00000001:0.0:1775777073.505321:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777073.505322:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232288714240 : -114841420837376 : ffff978d65e7ce00) 00000100:00000040:0.0:1775777073.505328:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505334:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777073.505336:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777073.505338:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777073.505341:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099830784, offset 0 00000400:00000200:0.0:1775777073.505344:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777073.505349:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.505352:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777073.505355:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777073.505358:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777073.505360:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777073.505362:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777073.505365:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4721:4721:256:4294967295] 192.168.201.139@tcp LPNI seq info [4721:4721:8:4294967295] 00000400:00000200:0.0:1775777073.505369:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777073.505374:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777073.505379:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.505383:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777073.505387:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777073.505391:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775777073.505395:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777073.505398:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775777073.505416:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777073.505418:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777073.505421:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777073.505425:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777073.505428:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777073.505430:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777073.505432:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777073.505436:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505443:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.505446:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505451:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.505452:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777073.505456:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777073.505459:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777073.505462:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777073.505464:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313461258 00000100:00000001:0.0:1775777073.505466:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313462518 00000100:00000001:0.0:1775777073.505468:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313461258 00000100:00000001:0.0:1775777073.505470:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313462518 00000100:00000001:0.0:1775777073.505472:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777073.505474:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.505476:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777073.505479:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777073/real 1775777073] req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777073.505487:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777073.505488:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777073.505490:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777073.505492:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777073.505494:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777073.505496:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777073.505499:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777073.505507:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505512:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777073.505514:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505515:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.505516:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777073.505519:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505524:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777073.505527:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505533:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777073.505535:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505536:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777073.505537:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505539:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777073.505540:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777073.505542:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777073.505544:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777073.505546:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777073.505549:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777073.505556:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65e7ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099830784:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777073.505559:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505561:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777073.505563:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65e7ce00 x1862033099830784/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777073.505569:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777073.505571:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777073.505572:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775777073.505574:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777073.505576:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777073.505577:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777073.505579:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777073.505580:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51506800. 02000000:00000001:0.0:1775777073.505582:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777073.505583:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777073.505584:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65e7ce00. 00000100:00000001:0.0:1775777073.505587:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777073.505588:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505590:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777073.505593:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777073.505594:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777073.505596:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777073.505597:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777073.692172:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777073.692179:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777073.692180:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777073.692186:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777073.692189:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777073.692193:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777073.692195:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777073.692198:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777073.692201:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777074.204261:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777074.204269:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777074.204272:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777074.204278:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777074.204282:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777074.204285:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777074.204288:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777074.204290:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777074.204293:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777074.268034:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777074.268041:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777074.268134:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777074.268135:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777074.268138:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777074.268140:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777074.268143:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777074.268144:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777074.271706:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777074.271714:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777074.271720:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777074.716125:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777074.716132:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777074.716133:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777074.716140:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777074.716144:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777074.716147:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777074.716150:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777074.716152:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777074.716155:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777074.780247:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313462518, at 4313462528 00000100:00000010:1.0:1775777074.780265:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237d80. 00000100:00000040:0.0:1775777074.781169:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313462518, at 4313462529 00000100:00000010:0.0:1775777074.781179:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775777075.227909:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777075.227916:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777075.227918:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777075.227923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777075.227927:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777075.227931:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777075.227934:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777075.227936:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777075.227939:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777075.291930:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777075.291935:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777075.291942:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777075.291943:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777075.291946:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777075.291947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777075.291950:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777075.291952:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777075.298676:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777075.298683:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777075.298688:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777075.740454:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777075.740461:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777075.740463:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777075.740469:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777075.740473:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777075.740477:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777075.740479:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777075.740481:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777075.740484:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777076.254719:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777076.254728:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777076.254730:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777076.254735:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777076.254740:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777076.254744:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777076.254747:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777076.254749:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777076.254751:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777076.316070:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777076.316076:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777076.316082:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777076.316083:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777076.316087:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777076.316088:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777076.316090:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777076.316091:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777076.316200:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777076.316204:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777076.316207:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777076.764184:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777076.764191:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777076.764193:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777076.764200:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777076.764203:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777076.764207:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777076.764210:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777076.764213:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777076.764216:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777077.279625:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777077.279632:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777077.279634:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777077.279639:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777077.279642:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777077.279647:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777077.279649:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777077.279652:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777077.279655:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777077.340542:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777077.340548:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777077.340553:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777077.340554:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777077.340557:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777077.340558:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777077.340561:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777077.340562:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777077.340976:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777077.340980:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777077.340983:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777077.787932:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777077.787939:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777077.787941:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777077.787947:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777077.787951:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777077.787955:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777077.787957:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777077.787960:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777077.787963:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777078.300475:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777078.300482:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777078.300484:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777078.300490:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777078.300493:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777078.300498:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777078.300501:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777078.300503:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777078.300506:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777078.364293:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777078.364300:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777078.364306:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777078.364307:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777078.364310:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777078.364311:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777078.364314:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777078.364315:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777078.364430:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777078.364434:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777078.364438:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775777078.621298:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777078.621309:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775777078.621312:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777078.621316:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777078.621319:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777078.621338:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18793 00000400:00000200:1.0:1775777078.621347:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777078.621353:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777078.621356:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777078.621359:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621370:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621373:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621376:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777078.621380:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777078.621382:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775777078.621385:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777078.621388:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775777078.621390:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775777078.621392:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775777078.621395:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621396:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621399:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621401:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777078.621403:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775777078.621406:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621409:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777078.621412:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777078.621415:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777078.621418:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621420:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775777078.621423:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777078.621432:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d180. 00000020:00000040:1.0:1775777078.621436:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775777078.621440:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777078.621442:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621445:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777078.621447:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777078.621452:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777078.621459:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3bc00. 00000100:00000001:1.0:1775777078.621466:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777078.621475:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777078.621485:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777078.621489:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777078.621494:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000021f8d018] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777078.621522:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777078.621526:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777078.621529:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775777078.621530:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777078.621532:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777078.621534:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777078.621537:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18793 00000400:00000200:1.0:1775777078.621541:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777078.621544:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777078.621546:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777078.621549:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621551:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621552:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621554:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777078.621557:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777078.621558:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775777078.621560:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777078.621562:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775777078.621563:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775777078.621565:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775777078.621568:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621570:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621571:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621573:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777078.621575:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775777078.621578:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621580:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777078.621582:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777078.621587:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777078.621588:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621590:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777078.621594:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775777078.621596:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621598:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777078.621601:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ed80. 00000020:00000040:1.0:1775777078.621604:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775777078.621606:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777078.621608:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621609:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777078.621611:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777078.621623:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777078.621628:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39000. 00000100:00000001:1.0:1775777078.621631:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777078.621636:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777078.621643:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777078.621645:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777078.621649:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008314c48f] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777078.621656:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777078.621659:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777078.621662:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775777078.621664:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777078.621665:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777078.621667:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777078.621669:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18793 00000400:00000200:1.0:1775777078.621673:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777078.621676:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777078.621678:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777078.621680:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621682:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621684:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621686:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777078.621688:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777078.621689:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775777078.621690:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777078.621692:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775777078.621694:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775777078.621696:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775777078.621699:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621701:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621702:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621704:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777078.621706:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775777078.621709:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621711:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777078.621713:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777078.621716:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777078.621718:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621731:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777078.621733:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775777078.621735:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621737:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777078.621740:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ce00. 00000020:00000040:1.0:1775777078.621743:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775777078.621745:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777078.621746:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621748:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777078.621749:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777078.621751:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777078.621754:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775777078.621756:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777078.621761:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777078.621768:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777078.621770:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777078.621773:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000436d95e2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777078.621807:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777078.621810:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777078.621814:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775777078.621815:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777078.621816:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777078.621819:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777078.621821:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18793 00000400:00000200:1.0:1775777078.621835:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777078.621838:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777078.621840:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777078.621843:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621845:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621847:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621849:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777078.621851:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777078.621852:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775777078.621853:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777078.621855:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775777078.621857:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775777078.621859:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775777078.621862:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621864:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621865:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621867:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777078.621869:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775777078.621871:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621873:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777078.621875:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777078.621878:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777078.621880:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621882:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777078.621885:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775777078.621888:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621889:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777078.621892:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6c000. 00000020:00000040:1.0:1775777078.621894:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775777078.621896:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777078.621897:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621899:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777078.621900:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777078.621901:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777078.621904:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775777078.621907:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777078.621912:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777078.621918:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777078.621920:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777078.621923:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb413703] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777078.621931:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777078.621933:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777078.621936:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775777078.621937:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777078.621938:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777078.621941:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777078.621943:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18793 00000400:00000200:1.0:1775777078.621946:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777078.621949:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777078.621951:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777078.621953:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621955:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621956:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621959:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777078.621960:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777078.621962:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775777078.621963:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777078.621965:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775777078.621966:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775777078.621968:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775777078.621971:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621972:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777078.621974:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621975:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777078.621977:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775777078.621979:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777078.621981:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777078.621983:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777078.621985:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777078.621987:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621988:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777078.621990:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775777078.621992:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.621993:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777078.621996:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6f480. 00000020:00000040:1.0:1775777078.621998:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775777078.622000:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777078.622001:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777078.622003:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777078.622004:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777078.622006:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777078.622009:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775777078.622012:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777078.622016:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777078.622027:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777078.622029:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777078.622033:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000057672ef1] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777078.622040:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777078.622044:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18804) 00000100:00000001:0.0:1775777078.622120:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777078.622124:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777078.622127:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777078.622133:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622145:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777078.622147:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.622150:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777078.622153:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777078.622155:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777078.622158:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099830912:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777078.622162:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777078.622165:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777078.622167:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.622170:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777078.622175:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775777078.622177:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777078.622182:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777078.622186:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777078.622192:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777078.622200:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099830912, portal 25 00000100:00000001:0.0:1775777078.622202:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777078.622203:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601024 : -114841421950592 : ffff978d65d6d180) 00000100:00000040:0.0:1775777078.622209:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622216:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777078.622218:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777078.622220:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777078.622223:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099830912, offset 0 00000400:00000200:0.0:1775777078.622229:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777078.622236:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.622241:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777078.622246:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777078.622250:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777078.622253:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777078.622256:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777078.622260:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4722:4722:256:4294967295] 192.168.201.139@tcp LPNI seq info [4722:4722:8:4294967295] 00000400:00000200:0.0:1775777078.622264:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.622274:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777078.622279:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.622284:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777078.622290:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777078.622294:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777078.622299:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777078.622302:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777078.622306:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.622309:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777078.622315:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777078.622319:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777078.622324:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777078.622327:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777078.622331:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777078.622334:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622343:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.622346:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622353:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.622354:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777078.622359:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777078.622362:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777078.622365:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313467639 00000100:00000010:0.0:1775777078.622370:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00000001:0.0:1775777078.622377:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775777078.622383:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313467639 00000100:00000010:0.0:1775777078.622386:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237000. 00000100:00000001:0.0:1775777078.622389:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775777078.622392:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777078.622394:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.622397:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777078.622400:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777078/real 1775777078] req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777078.622410:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777078.622411:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777078.622413:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777078.622415:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777078.622417:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777078.622419:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777078.622422:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777078.622429:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622434:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777078.622435:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.622437:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.622439:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777078.622442:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622447:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777078.622451:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622457:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777078.622459:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.622460:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.622462:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.622464:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777078.622466:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777078.622468:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777078.622472:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777078.622474:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777078.622478:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777078.622485:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099830912:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777078.622489:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777078.622492:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622497:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777078.622499:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.622501:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777078.622502:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777078.622504:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777078.622506:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831040:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777078.622509:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777078.622510:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777078.622512:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.622513:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777078.622516:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777078.622518:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777078.622521:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777078.622524:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777078.622527:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777078.622530:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099831040, portal 10 00000100:00000001:0.0:1775777078.622532:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777078.622533:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287608192 : -114841421943424 : ffff978d65d6ed80) 00000100:00000040:0.0:1775777078.622539:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622545:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777078.622547:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777078.622549:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777078.622552:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099831040, offset 0 00000400:00000200:0.0:1775777078.622555:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777078.622559:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.622562:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777078.622564:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777078.622567:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777078.622569:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777078.622571:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777078.622574:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4723:4723:256:4294967295] 192.168.201.139@tcp LPNI seq info [4723:4723:8:4294967295] 00000400:00000200:0.0:1775777078.622577:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.622582:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777078.622586:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.622589:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777078.622592:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777078.622594:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777078.622598:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777078.622600:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777078.622603:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.622604:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777078.622607:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777078.622610:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777078.622612:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777078.622614:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777078.622616:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777078.622619:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622626:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.622629:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622635:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.622636:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777078.622639:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777078.622643:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777078.622646:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777078.622648:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313466374 00000100:00000001:0.0:1775777078.622650:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313467639 00000100:00000001:0.0:1775777078.622653:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313466374 00000100:00000001:0.0:1775777078.622654:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313467639 00000100:00000001:0.0:1775777078.622656:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777078.622658:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.622659:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777078.622663:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777078/real 1775777078] req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777078.622669:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777078.622670:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777078.622672:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777078.622674:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777078.622675:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777078.622677:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777078.622680:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777078.622687:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622692:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777078.622693:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.622695:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.622696:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777078.622698:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622704:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777078.622707:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622715:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777078.622717:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.622730:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.622732:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.622734:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777078.622736:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777078.622738:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777078.622742:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777078.622743:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777078.622747:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777078.622753:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831040:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777078.622758:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777078.622760:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622766:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777078.622768:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.622770:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777078.622771:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777078.622773:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777078.622776:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831168:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777078.622817:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777078.622818:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777078.622820:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.622821:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777078.622825:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777078.622826:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777078.622829:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777078.622832:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777078.622835:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777078.622838:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099831168, portal 10 00000100:00000001:0.0:1775777078.622840:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777078.622841:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287600128 : -114841421951488 : ffff978d65d6ce00) 00000100:00000040:0.0:1775777078.622847:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622853:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777078.622855:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777078.622857:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777078.622859:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099831168, offset 0 00000400:00000200:0.0:1775777078.622863:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777078.622866:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.622870:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777078.622872:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777078.622875:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777078.622877:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777078.622879:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777078.622882:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4724:4724:256:4294967295] 192.168.201.139@tcp LPNI seq info [4724:4724:8:4294967295] 00000400:00000200:0.0:1775777078.622886:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.622892:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777078.622896:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.622899:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777078.622902:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777078.622905:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777078.622909:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777078.622911:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777078.622915:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.622917:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777078.622920:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777078.622922:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777078.622925:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777078.622927:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777078.622929:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777078.622932:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622939:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.622941:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.622947:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.622948:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777078.622951:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777078.622954:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777078.622957:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777078.622959:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313466376 00000100:00000001:0.0:1775777078.622961:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313467639 00000100:00000001:0.0:1775777078.622963:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313466376 00000100:00000001:0.0:1775777078.622965:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313467639 00000100:00000001:0.0:1775777078.622967:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777078.622969:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.622971:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777078.622974:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777078/real 1775777078] req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777078.622981:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777078.622982:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777078.622983:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777078.622986:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777078.622987:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777078.622989:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777078.622992:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777078.622999:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623006:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777078.623007:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623008:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.623010:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777078.623013:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623019:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777078.623022:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623029:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777078.623031:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623033:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.623034:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623035:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777078.623036:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777078.623039:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777078.623041:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777078.623043:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777078.623046:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777078.623053:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831168:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777078.623056:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777078.623059:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623066:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777078.623067:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.623070:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777078.623071:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777078.623073:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777078.623076:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831296:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777078.623079:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777078.623080:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777078.623082:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.623083:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777078.623086:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775777078.623088:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777078.623091:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777078.623093:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777078.623096:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777078.623098:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099831296, portal 4 00000100:00000001:0.0:1775777078.623100:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777078.623101:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287596544 : -114841421955072 : ffff978d65d6c000) 00000100:00000040:0.0:1775777078.623107:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623114:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777078.623116:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777078.623117:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777078.623119:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099831296, offset 0 00000400:00000200:0.0:1775777078.623123:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777078.623140:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.623144:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777078.623146:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777078.623149:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777078.623151:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777078.623153:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777078.623156:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4725:4725:256:4294967295] 192.168.201.139@tcp LPNI seq info [4725:4725:8:4294967295] 00000400:00000200:0.0:1775777078.623160:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.623165:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777078.623170:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.623173:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777078.623176:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777078.623178:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777078.623182:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777078.623184:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777078.623187:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.623189:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777078.623192:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777078.623195:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777078.623197:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777078.623199:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777078.623201:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777078.623204:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623211:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.623213:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623219:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.623221:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777078.623223:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777078.623225:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777078.623228:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777078.623230:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313466385 00000100:00000001:0.0:1775777078.623233:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313467639 00000100:00000001:0.0:1775777078.623234:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313466385 00000100:00000001:0.0:1775777078.623236:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313467639 00000100:00000001:0.0:1775777078.623238:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777078.623240:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.623242:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777078.623245:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777078/real 1775777078] req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777078.623253:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777078.623254:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777078.623256:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777078.623257:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777078.623259:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777078.623261:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777078.623263:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777078.623271:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623277:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777078.623278:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623279:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.623280:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777078.623283:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623288:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777078.623291:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623297:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777078.623299:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623300:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.623301:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623303:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777078.623305:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777078.623307:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777078.623309:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777078.623312:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777078.623315:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777078.623322:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831296:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777078.623326:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777078.623328:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623335:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777078.623336:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.623338:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777078.623340:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777078.623341:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777078.623344:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831424:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777078.623346:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777078.623348:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777078.623349:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777078.623351:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777078.623354:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777078.623356:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777078.623358:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777078.623361:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777078.623363:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777078.623366:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099831424, portal 4 00000100:00000001:0.0:1775777078.623367:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777078.623369:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287609984 : -114841421941632 : ffff978d65d6f480) 00000100:00000040:0.0:1775777078.623373:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623379:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777078.623381:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777078.623382:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777078.623384:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099831424, offset 0 00000400:00000200:0.0:1775777078.623388:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777078.623391:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.623394:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777078.623396:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777078.623398:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777078.623401:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777078.623403:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777078.623405:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4726:4726:256:4294967295] 192.168.201.139@tcp LPNI seq info [4726:4726:8:4294967295] 00000400:00000200:0.0:1775777078.623409:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777078.623414:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777078.623418:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.623420:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777078.623424:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777078.623426:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775777078.623429:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777078.623431:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775777078.623435:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777078.623437:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777078.623439:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777078.623442:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777078.623444:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777078.623446:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777078.623448:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777078.623451:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623457:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.623460:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623466:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.623467:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777078.623469:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777078.623471:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777078.623474:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777078.623476:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313466376 00000100:00000001:0.0:1775777078.623478:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313467639 00000100:00000001:0.0:1775777078.623480:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313466376 00000100:00000001:0.0:1775777078.623482:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313467639 00000100:00000001:0.0:1775777078.623483:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777078.623485:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.623487:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777078.623490:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777078/real 1775777078] req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777078.623497:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777078.623498:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777078.623499:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777078.623501:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777078.623503:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777078.623505:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777078.623507:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777078.623514:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623520:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777078.623521:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623523:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.623524:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777078.623527:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623534:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777078.623537:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623542:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777078.623544:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623545:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777078.623546:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623548:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777078.623549:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777078.623551:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777078.623554:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777078.623556:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777078.623559:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777078.623565:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831424:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777078.623569:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623571:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.623574:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d180 x1862033099830912/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623580:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777078.623582:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777078.623585:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e800. 02000000:00000001:0.0:1775777078.623588:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777078.623590:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777078.623592:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777078.623594:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777078.623596:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3bc00. 02000000:00000001:0.0:1775777078.623599:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777078.623601:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777078.623603:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d180. 00000100:00000001:0.0:1775777078.623606:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777078.623608:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623609:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.623612:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ed80 x1862033099831040/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623622:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777078.623623:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777078.623625:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777078.623626:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777078.623627:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777078.623628:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777078.623629:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777078.623631:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39000. 02000000:00000001:0.0:1775777078.623632:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777078.623634:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777078.623635:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ed80. 00000100:00000001:0.0:1775777078.623637:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777078.623638:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623640:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.623643:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ce00 x1862033099831168/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623648:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777078.623649:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777078.623650:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777078.623652:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777078.623653:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777078.623654:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777078.623656:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777078.623657:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b000. 02000000:00000001:0.0:1775777078.623658:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777078.623659:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777078.623660:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ce00. 00000100:00000001:0.0:1775777078.623662:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777078.623663:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623664:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.623667:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6c000 x1862033099831296/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623671:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777078.623672:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777078.623674:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775777078.623676:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777078.623676:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777078.623678:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777078.623680:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777078.623681:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a000. 02000000:00000001:0.0:1775777078.623683:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777078.623684:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777078.623685:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6c000. 00000100:00000001:0.0:1775777078.623687:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777078.623688:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623689:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777078.623692:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6f480 x1862033099831424/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777078.623697:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777078.623698:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777078.623699:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777078.623701:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777078.623702:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777078.623703:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777078.623704:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777078.623706:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39c00. 02000000:00000001:0.0:1775777078.623708:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777078.623709:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777078.623710:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6f480. 00000100:00000001:0.0:1775777078.623712:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777078.623713:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623715:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777078.623732:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777078.623735:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777078.623737:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777078.623739:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777078.814241:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777078.814250:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777078.814252:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777078.814258:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777078.814262:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777078.814266:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777078.814268:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777078.814271:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777078.814274:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777079.323898:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777079.323905:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777079.323907:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777079.323911:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777079.323914:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777079.323917:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777079.323919:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777079.323921:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777079.323923:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777079.389216:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777079.389224:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777079.389227:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777079.389231:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777079.389236:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777079.389239:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777079.389241:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777079.389244:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777079.389248:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777079.389250:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777079.389254:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777079.389255:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777079.389259:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777079.389261:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777079.389355:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777079.389360:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777079.389363:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777079.836108:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777079.836114:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777079.836116:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777079.836122:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777079.836126:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777079.836129:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777079.836132:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777079.836134:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777079.836136:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777079.899943:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313467639, at 4313467648 00000100:00000010:0.0:1775777079.899960:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000040:1.0:1775777079.901272:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313467639, at 4313467649 00000100:00000010:1.0:1775777079.901288:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237000. 00000100:00000001:3.0:1775777080.351134:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777080.351142:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777080.351144:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777080.351150:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777080.351154:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777080.351159:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777080.351162:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777080.351164:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777080.351168:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777080.411955:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777080.411960:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777080.411966:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777080.411968:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777080.411971:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777080.411972:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777080.411976:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777080.411977:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777080.412082:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777080.412085:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777080.412088:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777080.859980:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777080.859993:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777080.859995:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777080.860000:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777080.860003:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777080.860007:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777080.860015:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777080.860017:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777080.860020:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777081.371929:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777081.371936:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777081.371937:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777081.371943:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777081.371946:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777081.371950:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777081.371952:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777081.371954:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777081.371957:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777081.442281:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777081.442287:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777081.442292:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777081.442293:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777081.442296:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777081.442297:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777081.442300:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777081.442301:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777081.446238:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777081.446244:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777081.446248:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777081.884144:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777081.884152:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777081.884154:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777081.884160:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777081.884164:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777081.884168:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777081.884170:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777081.884173:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777081.884175:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777082.395961:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777082.395969:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777082.395972:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777082.395978:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777082.395982:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777082.395986:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777082.395989:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777082.395991:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777082.395994:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462945:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777082.462952:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462957:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777082.462960:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777082.462963:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777082.462966:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462969:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777082.462971:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462974:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462976:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462979:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777082.462981:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777082.462983:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777082.462984:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462986:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777082.462987:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462988:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462990:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462994:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777082.462996:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777082.462997:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777082.462998:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.462999:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777082.463000:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.463002:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777082.463003:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777082.463962:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777082.463966:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777082.463970:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777082.908068:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777082.908075:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777082.908077:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777082.908083:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777082.908087:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777082.908091:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777082.908094:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777082.908096:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777082.908099:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.420227:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777083.420235:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777083.420236:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777083.420242:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777083.420246:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777083.420250:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777083.420253:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777083.420255:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777083.420258:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777083.483964:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777083.483970:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777083.483976:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777083.483978:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777083.483981:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777083.483982:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777083.483985:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777083.483986:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777083.484247:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777083.484250:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777083.484253:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775777083.740355:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777083.740365:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775777083.740369:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777083.740372:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777083.740377:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777083.740384:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18799 00000400:00000200:3.0:1775777083.740393:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777083.740400:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777083.740404:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777083.740407:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740410:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777083.740412:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740414:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777083.740418:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777083.740420:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:3.0:1775777083.740423:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777083.740425:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:3.0:1775777083.740428:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:3.0:1775777083.740431:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:3.0:1775777083.740434:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740436:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777083.740438:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740440:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777083.740443:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:3.0:1775777083.740446:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740449:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777083.740452:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777083.740455:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777083.740458:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.740460:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775777083.740463:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777083.740473:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dedc00. 00000020:00000040:3.0:1775777083.740477:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:3.0:1775777083.740481:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777083.740482:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.740486:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777083.740489:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777083.740493:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777083.740501:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115ac00. 00000100:00000001:3.0:1775777083.740507:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777083.740516:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775777083.740526:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775777083.740530:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777083.740534:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000987bc73c] to pc [ptlrpcd_rcv+-1] req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777083.740563:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777083.740567:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777083.740570:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775777083.740572:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777083.740573:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777083.740575:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777083.740578:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18799 00000400:00000200:3.0:1775777083.740581:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777083.740584:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777083.740586:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777083.740589:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740591:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777083.740592:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740594:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777083.740597:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777083.740598:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:3.0:1775777083.740599:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775777083.740601:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:3.0:1775777083.740603:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:3.0:1775777083.740604:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:3.0:1775777083.740607:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740608:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777083.740610:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740611:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777083.740613:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:3.0:1775777083.740616:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740618:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777083.740620:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777083.740625:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777083.740628:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.740662:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777083.740666:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777083.740670:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.740673:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777083.740677:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def480. 00000020:00000040:3.0:1775777083.740680:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 00000100:00000001:0.0:1775777083.740681:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1775777083.740684:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775777083.740686:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1775777083.740686:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.740689:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775777083.740690:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777083.740693:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777083.740695:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777083.740698:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775777083.740700:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158000. 00000100:00000001:3.0:1775777083.740706:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.740714:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775777083.740715:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777083.740717:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.740722:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777083.740726:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775777083.740728:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1775777083.740732:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.740732:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777083.740741:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099831552:192.168.201.139@tcp:250:kworker.0 00000100:00000040:3.0:1775777083.740741:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087ca8a8c] to pc [ptlrpcd_rcv+-1] req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.740752:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775777083.740756:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.740758:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1775777083.740761:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1775777083.740762:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.740768:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1775777083.740768:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775777083.740771:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777083.740774:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:0.0:1775777083.740775:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e000. 00000100:00000001:3.0:1775777083.740777:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777083.740810:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18799 00000400:00000200:3.0:1775777083.740814:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777083.740818:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777083.740820:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775777083.740823:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740825:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777083.740827:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740830:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 02000000:00000001:0.0:1775777083.740831:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775777083.740835:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777083.740837:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:3.0:1775777083.740839:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775777083.740840:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000040:3.0:1775777083.740842:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:3.0:1775777083.740845:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:3.0:1775777083.740847:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:3.0:1775777083.740851:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740853:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777083.740855:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740856:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777083.740859:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:3.0:1775777083.740862:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740864:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777083.740866:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775777083.740869:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777083.740871:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775777083.740872:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000100:00000001:3.0:1775777083.740873:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777083.740877:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775777083.740879:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.740881:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775777083.740882:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000010:3.0:1775777083.740886:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45dec700. 00000020:00000040:3.0:1775777083.740889:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 00000100:00000200:0.0:1775777083.740891:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099831552, portal 25 02000000:00000001:3.0:1775777083.740891:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777083.740894:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.740896:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775777083.740897:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775777083.740899:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751257088 : -114841958294528 : ffff978d45dedc00) 02000000:00000001:3.0:1775777083.740899:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775777083.740902:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777083.740907:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d51158800. 00000100:00000040:0.0:1775777083.740909:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777083.740911:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777083.740919:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777083.740921:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777083.740929:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777083.740934:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000001:3.0:1775777083.740938:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1775777083.740940:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099831552, offset 0 00000100:00000001:3.0:1775777083.740941:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777083.740948:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bcc195f] to pc [ptlrpcd_rcv+-1] req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775777083.740953:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000100:00000001:3.0:1775777083.740961:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775777083.740963:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775777083.740966:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00080000:3.0:1775777083.740969:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1775777083.740971:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777083.740973:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775777083.740976:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000100:00000001:3.0:1775777083.740977:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775777083.740982:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18799 00000400:00000200:0.0:1775777083.740985:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775777083.740988:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.740993:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:3.0:1775777083.740993:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777083.740998:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775777083.741002:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1775777083.741005:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000400:00000200:0.0:1775777083.741009:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741010:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777083.741014:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741019:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000400:00000200:0.0:1775777083.741020:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4727:4727:256:4294967295] 192.168.201.139@tcp LPNI seq info [4727:4727:8:4294967295] 00000020:00000001:3.0:1775777083.741022:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775777083.741025:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:3.0:1775777083.741028:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775777083.741029:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000020:00000040:3.0:1775777083.741031:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:3.0:1775777083.741034:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:3.0:1775777083.741037:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:3.0:1775777083.741043:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741045:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775777083.741048:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741049:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777083.741053:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000400:00000200:0.0:1775777083.741057:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000100:00080000:3.0:1775777083.741059:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741065:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777083.741067:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.0:1775777083.741071:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 02000000:00000001:3.0:1775777083.741071:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777083.741076:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775777083.741079:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000100:00000001:3.0:1775777083.741079:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777083.741081:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1775777083.741085:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.741087:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775777083.741089:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000010:3.0:1775777083.741093:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45deed80. 00000800:00000200:0.0:1775777083.741094:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000020:00000040:3.0:1775777083.741096:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:3.0:1775777083.741101:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775777083.741104:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775777083.741104:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000100:00000001:3.0:1775777083.741107:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777083.741109:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.1:1775777083.741109:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 02000000:00000001:3.0:1775777083.741113:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777083.741117:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115a000. 00000100:00000001:3.0:1775777083.741120:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1775777083.741128:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000100:00100000:3.0:1775777083.741128:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:0.0:1775777083.741132:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000100:00000001:3.0:1775777083.741140:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775777083.741142:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775777083.741142:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777083.741148:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6074702] to pc [ptlrpcd_rcv+-1] req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775777083.741152:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777083.741160:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000100:00000001:3.0:1775777083.741160:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775777083.741164:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00080000:3.0:1775777083.741164:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775777083.741169:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775777083.741171:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775777083.741171:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775777083.741174:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775777083.741177:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:0.0:1775777083.741179:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775777083.741182:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18799 00000400:00000200:3.0:1775777083.741189:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:3.0:1775777083.741195:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:3.0:1775777083.741198:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775777083.741199:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775777083.741203:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741207:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777083.741208:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775777083.741212:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741217:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:3.0:1775777083.741222:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775777083.741224:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775777083.741224:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:3.0:1775777083.741227:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775777083.741229:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:3.0:1775777083.741230:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:3.0:1775777083.741233:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:3.0:1775777083.741236:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000001:0.0:1775777083.741237:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775777083.741240:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741243:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775777083.741245:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000040:3.0:1775777083.741247:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777083.741249:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313472758 00000100:00000001:3.0:1775777083.741252:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:3.0:1775777083.741255:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00000010:0.0:1775777083.741257:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237ea0. 00000100:00080000:3.0:1775777083.741258:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:3.0:1775777083.741263:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777083.741265:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775777083.741266:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1775777083.741268:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775777083.741271:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.741273:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313472758 00000100:00000001:3.0:1775777083.741273:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775777083.741276:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000100:00000010:0.0:1775777083.741278:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d6f237720. 00000008:00000001:3.0:1775777083.741280:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.741283:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775777083.741283:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775777083.741287:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d45def100. 00000100:00000001:0.0:1775777083.741288:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775777083.741291:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 00000100:00000001:0.0:1775777083.741292:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775777083.741296:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775777083.741297:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1775777083.741299:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.741301:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775777083.741304:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1775777083.741305:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777083/real 1775777083] req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775777083.741306:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775777083.741312:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d5115b000. 00000100:00000001:3.0:1775777083.741317:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775777083.741322:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:3.0:1775777083.741324:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1775777083.741325:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777083.741328:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777083.741334:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000100:00000001:3.0:1775777083.741337:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1775777083.741338:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:3.0:1775777083.741341:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.741342:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1775777083.741346:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcaec7c3] to pc [ptlrpcd_rcv+-1] req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777083.741350:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775777083.741367:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775777083.741371:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775777083.741374:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18809) 00000100:00000001:0.0:1775777083.741382:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777083.741383:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741386:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.741388:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777083.741393:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741399:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777083.741403:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741410:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777083.741413:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741415:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.741416:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741418:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777083.741420:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777083.741423:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777083.741426:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777083.741428:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777083.741431:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777083.741440:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099831552:192.168.201.139@tcp:250:kworker.0 00000100:00000001:0.0:1775777083.741444:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741446:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.741448:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dedc00 x1862033099831552/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741455:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777083.741457:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777083.741460:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777083.741463:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777083.741465:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777083.741466:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:0.0:1775777083.741468:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777083.741470:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115ac00. 02000000:00000001:0.0:1775777083.741473:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777083.741475:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777083.741476:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dedc00. 00000100:00000001:0.0:1775777083.741479:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777083.741480:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741482:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741489:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777083.741490:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.741493:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777083.741494:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775777083.741495:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777083.741499:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741506:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777083.741507:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.741509:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777083.741511:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777083.741513:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777083.741515:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831680:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777083.741519:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777083.741521:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777083.741522:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.741524:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777083.741527:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777083.741529:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777083.741531:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777083.741534:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777083.741538:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777083.741540:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099831680, portal 10 00000100:00000001:0.0:1775777083.741542:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777083.741543:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751263360 : -114841958288256 : ffff978d45def480) 00000100:00000040:0.0:1775777083.741549:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741555:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777083.741557:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777083.741559:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777083.741561:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099831680, offset 0 00000400:00000200:0.0:1775777083.741565:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777083.741569:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.741573:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777083.741575:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777083.741578:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777083.741580:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777083.741583:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777083.741586:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4728:4728:256:4294967295] 192.168.201.139@tcp LPNI seq info [4728:4728:8:4294967295] 00000400:00000200:0.0:1775777083.741590:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.741596:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777083.741600:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777083.741602:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777083.741606:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777083.741609:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777083.741613:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777083.741615:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777083.741619:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777083.741620:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777083.741623:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777083.741627:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777083.741641:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777083.741643:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777083.741646:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777083.741649:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741657:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.741660:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741666:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.741667:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777083.741670:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777083.741674:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:0.0:1775777083.741678:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777083.741680:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313471493 00000100:00000001:0.0:1775777083.741682:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313472758 00000100:00000001:0.0:1775777083.741684:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313471493 00000100:00000001:0.0:1775777083.741686:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313472758 00000100:00000001:0.0:1775777083.741688:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777083.741690:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.741692:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777083.741695:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777083/real 1775777083] req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777083.741703:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777083.741704:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777083.741705:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777083.741708:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777083.741709:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777083.741711:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777083.741714:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777083.741721:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741727:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777083.741728:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741730:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.741731:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777083.741735:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741741:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777083.741744:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741751:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777083.741753:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741755:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.741756:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.741758:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777083.741760:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777083.741763:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777083.741766:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777083.741768:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777083.741771:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777083.741798:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831680:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777083.741802:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777083.741805:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741810:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777083.741811:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.741813:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777083.741815:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777083.741817:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777083.741819:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831808:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777083.741822:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777083.741824:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777083.741825:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.741826:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777083.741829:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775777083.741831:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777083.741834:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777083.741836:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777083.741839:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777083.741842:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099831808, portal 10 00000100:00000001:0.0:1775777083.741843:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777083.741845:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751251712 : -114841958299904 : ffff978d45dec700) 00000100:00000040:0.0:1775777083.741849:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741860:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777083.741862:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777083.741863:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777083.741865:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099831808, offset 0 00000400:00000200:0.0:1775777083.741868:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777083.741871:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.741874:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777083.741876:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777083.741878:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777083.741881:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777083.741883:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777083.741885:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4729:4729:256:4294967295] 192.168.201.139@tcp LPNI seq info [4729:4729:8:4294967295] 00000400:00000200:0.0:1775777083.741889:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.741894:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777083.741898:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777083.741901:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777083.741904:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777083.741907:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777083.741910:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777083.741913:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777083.741916:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777083.741918:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777083.741920:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777083.741923:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777083.741925:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777083.741927:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777083.741929:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777083.741932:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741938:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.741941:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.741951:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.741953:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777083.741955:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777083.741957:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:0.0:1775777083.741960:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777083.741962:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313471495 00000100:00000001:0.0:1775777083.741964:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313472758 00000100:00000001:0.0:1775777083.741966:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313471495 00000100:00000001:0.0:1775777083.741968:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313472758 00000100:00000001:0.0:1775777083.741970:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777083.741971:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.741973:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777083.741976:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777083/real 1775777083] req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777083.741983:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777083.741985:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777083.741986:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777083.741988:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777083.741989:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777083.741991:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777083.741994:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777083.742001:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742007:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777083.742009:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742010:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.742011:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777083.742014:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742020:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777083.742024:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742030:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777083.742032:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742034:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.742035:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742037:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777083.742038:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777083.742040:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777083.742043:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777083.742045:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777083.742049:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777083.742056:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831808:192.168.201.139@tcp:38:kworker.0 00000100:00000001:0.0:1775777083.742060:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777083.742063:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742069:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777083.742070:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.742072:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777083.742073:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777083.742075:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777083.742078:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831936:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777083.742080:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777083.742082:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777083.742084:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.742085:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777083.742088:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777083.742090:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777083.742093:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777083.742095:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777083.742098:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777083.742101:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099831936, portal 4 00000100:00000001:0.0:1775777083.742103:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777083.742104:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751261568 : -114841958290048 : ffff978d45deed80) 00000100:00000040:0.0:1775777083.742110:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742116:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777083.742118:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777083.742120:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777083.742122:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099831936, offset 0 00000400:00000200:0.0:1775777083.742126:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777083.742130:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.742133:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777083.742135:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777083.742137:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777083.742139:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777083.742141:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777083.742144:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4730:4730:256:4294967295] 192.168.201.139@tcp LPNI seq info [4730:4730:8:4294967295] 00000400:00000200:0.0:1775777083.742147:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.742153:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777083.742157:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777083.742160:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777083.742163:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777083.742165:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777083.742169:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777083.742171:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777083.742174:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777083.742176:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777083.742179:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777083.742182:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777083.742184:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777083.742186:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777083.742188:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777083.742191:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742198:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.742201:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742207:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.742208:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777083.742211:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777083.742214:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:0.0:1775777083.742216:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777083.742218:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313471504 00000100:00000001:0.0:1775777083.742220:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313472758 00000100:00000001:0.0:1775777083.742222:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313471504 00000100:00000001:0.0:1775777083.742224:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313472758 00000100:00000001:0.0:1775777083.742226:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777083.742228:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.742229:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777083.742232:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777083/real 1775777083] req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777083.742240:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777083.742241:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777083.742242:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777083.742244:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777083.742246:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777083.742248:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777083.742251:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777083.742259:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742264:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777083.742265:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742267:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.742268:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777083.742271:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742277:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777083.742280:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742286:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777083.742288:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742289:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.742291:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742292:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777083.742294:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777083.742297:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777083.742300:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777083.742302:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777083.742305:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777083.742312:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099831936:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777083.742315:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775777083.742318:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742325:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775777083.742326:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.742328:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775777083.742330:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777083.742331:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777083.742335:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832064:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777083.742337:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775777083.742339:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775777083.742340:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777083.742342:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775777083.742345:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777083.742347:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775777083.742350:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:0.0:1775777083.742352:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d513069a0. 00000400:00000010:0.0:1775777083.742356:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b8000. 00000100:00000200:0.0:1775777083.742358:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099832064, portal 4 00000100:00000001:0.0:1775777083.742360:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775777083.742361:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231751262464 : -114841958289152 : ffff978d45def100) 00000100:00000040:0.0:1775777083.742365:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742372:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775777083.742374:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:0.0:1775777083.742376:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d727b83b8. 00000100:00000200:0.0:1775777083.742378:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099832064, offset 0 00000400:00000200:0.0:1775777083.742382:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:0.0:1775777083.742386:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.742389:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:0.0:1775777083.742391:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775777083.742393:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:0.0:1775777083.742396:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775777083.742398:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:0.0:1775777083.742401:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4731:4731:256:4294967295] 192.168.201.139@tcp LPNI seq info [4731:4731:8:4294967295] 00000400:00000200:0.0:1775777083.742405:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777083.742410:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:0.0:1775777083.742414:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777083.742417:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978d71c9ba00. 00000800:00000200:0.0:1775777083.742421:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.0:1775777083.742423:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775777083.742427:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:0.1:1775777083.742429:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775777083.742432:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:0.0:1775777083.742434:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978d71c9ba00. 00000400:00000200:0.0:1775777083.742436:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775777083.742439:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775777083.742442:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b83b8 00000400:00000010:0.0:1775777083.742446:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b83b8. 00000100:00000001:0.0:1775777083.742449:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775777083.742452:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742458:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.742460:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742465:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.742466:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775777083.742468:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777083.742471:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:0.0:1775777083.742473:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:0.0:1775777083.742475:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313471495 00000100:00000001:0.0:1775777083.742477:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313472758 00000100:00000001:0.0:1775777083.742478:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313471495 00000100:00000001:0.0:1775777083.742480:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313472758 00000100:00000001:0.0:1775777083.742482:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775777083.742483:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.742485:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775777083.742488:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777083/real 1775777083] req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775777083.742494:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775777083.742495:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775777083.742497:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d513069a0. 00000400:00000200:0.0:1775777083.742499:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d727b8000 00000400:00000010:0.0:1775777083.742500:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d727b8000. 00000100:00000001:0.0:1775777083.742501:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775777083.742504:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775777083.742510:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742515:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775777083.742516:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742518:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.742519:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775777083.742522:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742527:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775777083.742530:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742536:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775777083.742538:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742539:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775777083.742540:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742541:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775777083.742543:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775777083.742545:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775777083.742548:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:0.0:1775777083.742549:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775777083.742552:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775777083.742559:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d45def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832064:192.168.201.139@tcp:8:kworker.0 00000100:00000001:0.0:1775777083.742562:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742563:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.742566:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def480 x1862033099831680/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742571:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777083.742572:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777083.742574:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759e000. 02000000:00000001:0.0:1775777083.742577:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777083.742579:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777083.742580:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:0.0:1775777083.742582:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777083.742584:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158000. 02000000:00000001:0.0:1775777083.742586:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777083.742588:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777083.742589:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def480. 00000100:00000001:0.0:1775777083.742592:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777083.742593:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742594:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.742597:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45dec700 x1862033099831808/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742602:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777083.742603:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777083.742604:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759dc00. 02000000:00000001:0.0:1775777083.742606:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777083.742607:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777083.742608:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:0.0:1775777083.742610:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777083.742612:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d51158800. 02000000:00000001:0.0:1775777083.742613:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777083.742614:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777083.742615:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45dec700. 00000100:00000001:0.0:1775777083.742617:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777083.742618:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742620:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.742622:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45deed80 x1862033099831936/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742628:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777083.742629:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777083.742643:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759f800. 02000000:00000001:0.0:1775777083.742646:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777083.742647:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777083.742648:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:0.0:1775777083.742650:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777083.742651:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115a000. 02000000:00000001:0.0:1775777083.742653:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777083.742654:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777083.742656:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45deed80. 00000100:00000001:0.0:1775777083.742658:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777083.742659:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742661:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775777083.742664:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d45def100 x1862033099832064/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777083.742669:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775777083.742670:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775777083.742671:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d4759c800. 02000000:00000001:0.0:1775777083.742673:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775777083.742674:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775777083.742675:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:0.0:1775777083.742676:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775777083.742678:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d5115b000. 02000000:00000001:0.0:1775777083.742679:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775777083.742680:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775777083.742681:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d45def100. 00000100:00000001:0.0:1775777083.742683:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775777083.742684:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742686:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775777083.742691:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775777083.742693:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777083.742695:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775777083.742696:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777083.938530:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777083.938538:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777083.938540:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777083.938545:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777083.938550:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777083.938554:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777083.938557:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777083.938560:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777083.938563:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777084.443912:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777084.443920:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777084.443922:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777084.443929:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777084.443934:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777084.443938:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777084.443941:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777084.443944:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777084.443947:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777084.510510:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777084.510516:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777084.510522:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777084.510523:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777084.510526:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777084.510527:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777084.510530:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777084.510531:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777084.510654:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777084.510657:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777084.510660:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777084.957128:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777084.957134:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777084.957137:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777084.957141:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777084.957145:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777084.957150:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777084.957152:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777084.957155:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777084.957159:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777085.020488:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313472758, at 4313472768 00000100:00000010:1.0:1775777085.020514:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237720. 00000100:00000040:0.0:1775777085.020848:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313472758, at 4313472769 00000100:00000010:0.0:1775777085.020856:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d6f237ea0. 00000100:00000001:3.0:1775777085.467937:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777085.467943:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777085.467945:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777085.467951:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777085.467955:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777085.467958:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777085.467961:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777085.467976:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777085.467979:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777085.533718:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777085.533724:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777085.533730:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777085.533731:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777085.533734:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777085.533735:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777085.533739:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777085.533741:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777085.533851:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777085.533855:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777085.533858:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777085.980501:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777085.980508:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777085.980510:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777085.980515:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777085.980519:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777085.980522:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777085.980525:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777085.980527:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777085.980530:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777086.492701:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777086.492710:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777086.492713:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777086.492719:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777086.492722:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777086.492727:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777086.492729:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777086.492731:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777086.492734:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777086.556009:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777086.556015:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777086.556022:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777086.556023:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777086.556026:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777086.556028:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777086.556032:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777086.556033:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777086.556144:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777086.556147:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777086.556150:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777087.003931:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777087.003952:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777087.003954:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777087.003960:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777087.003963:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777087.003968:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777087.003972:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777087.003975:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777087.003978:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777087.516260:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777087.516267:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777087.516269:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777087.516274:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777087.516278:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777087.516281:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777087.516283:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777087.516286:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777087.516288:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777087.579935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777087.579940:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777087.579946:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777087.579947:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777087.579951:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777087.579952:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777087.579955:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777087.579956:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777087.580059:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777087.580061:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777087.580065:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777088.027941:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777088.027948:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777088.027950:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777088.027957:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777088.027961:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777088.027965:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777088.027967:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777088.027970:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777088.027973:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777088.539896:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777088.539904:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777088.539906:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777088.539912:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777088.539915:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777088.539918:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777088.539922:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777088.539924:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777088.539926:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777088.603980:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777088.603988:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777088.603991:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777088.603994:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777088.604000:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777088.604002:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777088.604004:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777088.604006:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777088.604011:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777088.604012:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777088.604015:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777088.604016:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777088.604020:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777088.604021:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777088.604167:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777088.604171:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777088.604174:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775777088.860189:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777088.860199:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775777088.860202:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777088.860206:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777088.860210:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777088.860217:0:303560:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18804 00000400:00000200:0.0:1775777088.860228:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777088.860234:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777088.860237:0:303560:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777088.860240:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860244:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.860246:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860249:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777088.860253:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777088.860255:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:0.0:1775777088.860257:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777088.860260:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:0.0:1775777088.860263:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:0.0:1775777088.860265:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:0.0:1775777088.860267:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860270:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.860272:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860274:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777088.860277:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:0.0:1775777088.860280:0:303560:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860282:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777088.860285:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777088.860289:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777088.860292:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.860294:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775777088.860296:0:303560:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777088.860307:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dd180. 00000020:00000040:0.0:1775777088.860311:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:0.0:1775777088.860314:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777088.860316:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.860319:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777088.860322:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777088.860326:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777088.860334:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759c800. 00000100:00000001:0.0:1775777088.860341:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777088.860349:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777088.860360:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777088.860364:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777088.860369:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c5a7de3e] to pc [ptlrpcd_rcv+-1] req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.860437:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775777088.860441:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775777088.860443:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777088.860449:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.860457:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777088.860459:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.860462:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777088.860465:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777088.860467:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777088.860470:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099832192:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775777088.860474:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777088.860477:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777088.860479:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.860482:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777088.860488:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775777088.860489:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777088.860494:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777088.860498:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777088.860518:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777088.860524:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099832192, portal 25 00000100:00000001:2.0:1775777088.860526:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777088.860528:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741751680 : -114841967799936 : ffff978d454dd180) 00000100:00000040:2.0:1775777088.860533:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.860550:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777088.860553:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777088.860556:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777088.860560:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099832192, offset 0 00000400:00000200:2.0:1775777088.860566:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777088.860572:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.860578:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777088.860582:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777088.860586:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777088.860589:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777088.860591:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777088.860595:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4732:4732:256:4294967295] 192.168.201.139@tcp LPNI seq info [4732:4732:8:4294967295] 00000400:00000200:2.0:1775777088.860607:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.860617:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777088.860624:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.860629:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777088.860635:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777088.860639:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775777088.860644:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777088.860647:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775777088.860651:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.860654:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777088.860660:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777088.860665:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777088.860669:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777088.860671:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777088.860675:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777088.860678:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777088.860726:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777088.860729:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777088.860732:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775777088.860733:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777088.860735:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777088.860737:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777088.860740:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18804 00000400:00000200:0.0:1775777088.860744:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777088.860746:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777088.860748:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777088.860751:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860753:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.860754:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860756:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777088.860758:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777088.860759:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:0.0:1775777088.860761:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777088.860763:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:0.0:1775777088.860764:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:0.0:1775777088.860766:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:0.0:1775777088.860768:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860770:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.860771:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860773:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777088.860775:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:0.0:1775777088.860995:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777088.860998:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777088.861000:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777088.861006:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777088.861008:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.861010:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777088.861014:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775777088.861017:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.861019:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777088.861023:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dca80. 00000020:00000040:0.0:1775777088.861026:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:0.0:1775777088.861029:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777088.861031:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.861032:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777088.861034:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777088.861036:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777088.861040:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759f800. 00000100:00000001:0.0:1775777088.861044:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777088.861048:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777088.861056:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777088.861058:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777088.861062:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000066043726] to pc [ptlrpcd_rcv+-1] req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.864584:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.864599:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.864611:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.864613:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777088.864628:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777088.864635:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777088.864639:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313477878 00000100:00000010:2.0:1775777088.864648:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b4e0. 00000100:00000001:2.0:1775777088.864654:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775777088.864661:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313477878 00000100:00000010:2.0:1775777088.864665:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b8a0. 00000100:00000001:2.0:1775777088.864668:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775777088.864671:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777088.864675:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.864677:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777088.864682:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777088/real 1775777088] req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777088.864700:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777088.864702:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777088.864706:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777088.864709:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777088.864711:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777088.864714:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777088.864718:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777088.864726:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.864734:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777088.864736:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.864738:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.864740:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777088.864743:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.864750:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777088.864754:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.864762:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777088.864765:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.864767:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.864768:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.864771:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777088.864773:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777088.864777:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777088.864795:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777088.864798:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777088.864802:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777088.864812:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099832192:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775777088.864816:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.864818:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.864821:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dd180 x1862033099832192/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.864827:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777088.864830:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777088.864834:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775777088.864839:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777088.864842:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777088.864844:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775777088.864845:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777088.864848:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759c800. 02000000:00000001:2.0:1775777088.864852:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777088.864854:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777088.864856:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dd180. 00000100:00000001:2.0:1775777088.864859:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777088.864861:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.864863:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.864872:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775777088.864873:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.864876:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775777088.864878:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775777088.864880:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777088.864884:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.864892:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777088.864894:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.864896:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777088.864899:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777088.864902:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777088.864905:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832320:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775777088.864909:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777088.864912:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777088.864915:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.864917:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777088.864922:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775777088.864924:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777088.864928:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777088.864932:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777088.864937:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777088.864943:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099832320, portal 10 00000100:00000001:2.0:1775777088.864945:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777088.864947:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741749888 : -114841967801728 : ffff978d454dca80) 00000100:00000040:2.0:1775777088.864954:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.864960:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777088.864963:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777088.864966:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777088.864969:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099832320, offset 0 00000400:00000200:2.0:1775777088.864975:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777088.864982:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.864988:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777088.864993:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777088.864997:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777088.865000:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777088.865003:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777088.865007:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4733:4733:256:4294967295] 192.168.201.139@tcp LPNI seq info [4733:4733:8:4294967295] 00000400:00000200:2.0:1775777088.865012:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.865021:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777088.865027:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.865033:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777088.865039:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777088.865044:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775777088.865049:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777088.865052:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775777088.865056:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.865058:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777088.865063:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777088.865067:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777088.865071:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777088.865072:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777088.865075:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777088.865078:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.865085:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.865087:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.865094:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.865095:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777088.865098:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777088.865101:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775777088.865103:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777088.865105:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313476613 00000100:00000001:2.0:1775777088.865108:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313477878 00000100:00000001:2.0:1775777088.865110:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313476613 00000100:00000001:2.0:1775777088.865111:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313477878 00000100:00000001:2.0:1775777088.865113:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777088.865115:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.865117:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777088.865120:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777088/real 1775777088] req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777088.865128:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777088.865129:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777088.865131:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777088.865134:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777088.865135:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777088.865137:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777088.865140:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777088.865148:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.865155:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777088.865156:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.865158:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.865159:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777088.865162:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.865169:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777088.865173:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.865179:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777088.865182:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.865183:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.865184:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.865186:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777088.865188:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777088.865190:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777088.865194:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777088.865196:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777088.865200:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777088.865207:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832320:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775777088.865211:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.865212:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.865215:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dca80 x1862033099832320/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.865221:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777088.865223:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777088.865224:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775777088.865227:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777088.865228:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777088.865229:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775777088.865231:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777088.865232:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759f800. 02000000:00000001:2.0:1775777088.865235:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777088.865236:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777088.865238:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dca80. 00000100:00000001:2.0:1775777088.865240:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777088.865241:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.865242:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.865246:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775777088.865248:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.865249:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775777088.865251:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.866771:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777088.866800:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777088.866806:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775777088.866808:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777088.866810:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777088.866813:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777088.866819:0:303560:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18804 00000400:00000200:0.0:1775777088.866828:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777088.866835:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777088.866837:0:303560:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777088.866840:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.866844:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.866846:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.866848:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777088.866851:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777088.866853:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:0.0:1775777088.866855:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777088.866858:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:0.0:1775777088.866860:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:0.0:1775777088.866863:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:0.0:1775777088.866866:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.866868:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.866869:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.866871:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777088.866874:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:0.0:1775777088.866876:0:303560:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777088.866879:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777088.866882:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777088.866888:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777088.866891:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.866893:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777088.866897:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775777088.866901:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.866902:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777088.866911:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de300. 00000020:00000040:0.0:1775777088.866914:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:0.0:1775777088.866918:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777088.866920:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.866922:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777088.866925:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777088.866928:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777088.866934:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759dc00. 00000100:00000001:0.0:1775777088.866940:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777088.866947:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777088.866956:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777088.866960:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777088.866964:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f29a8133] to pc [ptlrpcd_rcv+-1] req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777088.866973:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777088.866976:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777088.866978:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775777088.866979:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777088.866981:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777088.866983:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777088.866985:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18804 00000400:00000200:0.0:1775777088.866989:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777088.866991:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777088.866993:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777088.866995:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.866997:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.866999:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867001:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777088.867003:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777088.867004:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:0.0:1775777088.867006:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777088.867007:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:0.0:1775777088.867009:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:0.0:1775777088.867011:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:0.0:1775777088.867014:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867015:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.867018:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867019:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777088.867021:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:0.0:1775777088.867023:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867025:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777088.867027:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777088.867030:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777088.867032:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.867034:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777088.867036:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775777088.867039:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.867040:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777088.867044:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454dfb80. 00000020:00000040:0.0:1775777088.867046:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:0.0:1775777088.867048:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777088.867049:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.867050:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777088.867051:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777088.867053:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777088.867056:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e000. 00000100:00000001:0.0:1775777088.867059:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777088.867063:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777088.867070:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777088.867071:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777088.867075:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ef8ea28] to pc [ptlrpcd_rcv+-1] req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777088.867082:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775777088.867084:0:303560:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775777088.867088:0:303560:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1775777088.867089:0:303560:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775777088.867091:0:303560:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775777088.867093:0:303560:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775777088.867096:0:303560:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18804 00000400:00000200:0.0:1775777088.867099:0:303560:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:0.0:1775777088.867102:0:303560:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:0.0:1775777088.867104:0:303560:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775777088.867107:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867109:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.867111:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867113:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:0.0:1775777088.867115:0:303560:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775777088.867116:0:303560:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:0.0:1775777088.867118:0:303560:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775777088.867119:0:303560:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:0.0:1775777088.867121:0:303560:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:0.0:1775777088.867123:0:303560:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:0.0:1775777088.867125:0:303560:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867127:0:303560:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775777088.867129:0:303560:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867131:0:303560:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:0.0:1775777088.867133:0:303560:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:0.0:1775777088.867136:0:303560:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:0.0:1775777088.867138:0:303560:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775777088.867139:0:303560:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775777088.867141:0:303560:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775777088.867143:0:303560:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.867145:0:303560:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775777088.867146:0:303560:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1775777088.867148:0:303560:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.867149:0:303560:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775777088.867152:0:303560:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d454de680. 00000020:00000040:0.0:1775777088.867154:0:303560:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:0.0:1775777088.867156:0:303560:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775777088.867157:0:303560:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775777088.867158:0:303560:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775777088.867159:0:303560:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775777088.867161:0:303560:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775777088.867163:0:303560:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d4759e800. 00000100:00000001:0.0:1775777088.867167:0:303560:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775777088.867171:0:303560:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775777088.867182:0:303560:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775777088.867184:0:303560:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777088.867187:0:303560:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4001887] to pc [ptlrpcd_rcv+-1] req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775777088.867193:0:303560:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777088.867198:0:303560:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18814) 00000100:00000001:2.0:1775777088.867333:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775777088.867337:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775777088.867339:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777088.867345:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.867352:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777088.867354:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.867357:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777088.867359:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777088.867361:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777088.867364:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832448:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775777088.867368:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777088.867372:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777088.867374:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.867376:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777088.867381:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775777088.867383:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777088.867389:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777088.867392:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777088.867399:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777088.867405:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099832448, portal 10 00000100:00000001:2.0:1775777088.867407:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777088.867409:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741756160 : -114841967795456 : ffff978d454de300) 00000100:00000040:2.0:1775777088.867416:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.867423:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777088.867426:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777088.867428:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777088.867432:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099832448, offset 0 00000400:00000200:2.0:1775777088.867438:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777088.867445:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.867450:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777088.867455:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777088.867459:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777088.867463:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777088.867466:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777088.867469:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4734:4734:256:4294967295] 192.168.201.139@tcp LPNI seq info [4734:4734:8:4294967295] 00000400:00000200:2.0:1775777088.867474:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.867482:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777088.867487:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.867493:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777088.867499:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777088.867502:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775777088.867507:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777088.867510:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775777088.867515:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.867517:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777088.867523:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777088.867527:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777088.867531:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777088.867534:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777088.867537:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777088.867552:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.867561:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.867565:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.867571:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.867572:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777088.867577:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777088.867580:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775777088.867583:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777088.867586:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313476620 00000100:00000001:2.0:1775777088.867588:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313477878 00000100:00000001:2.0:1775777088.867590:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313476620 00000100:00000001:2.0:1775777088.867592:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313477878 00000100:00000001:2.0:1775777088.867594:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777088.868170:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.868174:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777088.868181:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777088/real 1775777088] req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777088.868192:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777088.868194:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777088.868196:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777088.868199:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777088.868201:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777088.868204:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777088.868207:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777088.868215:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868222:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777088.868224:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868226:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.868228:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777088.868231:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868237:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777088.868241:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868247:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777088.868249:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868251:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.868252:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868254:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777088.868256:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777088.868259:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777088.868262:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777088.868265:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777088.868268:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777088.868275:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832448:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775777088.868280:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777088.868283:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868288:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777088.868290:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.868292:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777088.868294:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777088.868296:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777088.868298:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832576:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775777088.868301:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777088.868303:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777088.868305:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.868307:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777088.868311:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506000. 02000000:00000001:2.0:1775777088.868313:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777088.868317:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777088.868320:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777088.868324:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777088.868327:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099832576, portal 4 00000100:00000001:2.0:1775777088.868328:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777088.868329:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741762432 : -114841967789184 : ffff978d454dfb80) 00000100:00000040:2.0:1775777088.868334:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868339:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777088.868341:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777088.868343:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777088.868346:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099832576, offset 0 00000400:00000200:2.0:1775777088.868350:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777088.868354:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.868359:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777088.868362:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777088.868365:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777088.868367:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777088.868369:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777088.868373:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4735:4735:256:4294967295] 192.168.201.139@tcp LPNI seq info [4735:4735:8:4294967295] 00000400:00000200:2.0:1775777088.868377:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.868384:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777088.868388:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.868392:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777088.868396:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777088.868399:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775777088.868403:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777088.868405:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775777088.868409:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.868411:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777088.868414:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777088.868417:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777088.868420:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777088.868422:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777088.868424:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777088.868428:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868435:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.868438:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868444:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.868446:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777088.868450:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777088.868453:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775777088.868456:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777088.868458:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313476630 00000100:00000001:2.0:1775777088.868460:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313477878 00000100:00000001:2.0:1775777088.868462:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313476630 00000100:00000001:2.0:1775777088.868464:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313477878 00000100:00000001:2.0:1775777088.868466:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777088.868469:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.868470:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777088.868474:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777088/real 1775777088] req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777088.868481:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777088.868482:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777088.868483:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777088.868485:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777088.868487:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777088.868489:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777088.868491:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777088.868513:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868520:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777088.868521:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868523:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.868524:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777088.868527:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868533:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777088.868537:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868553:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777088.868555:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868556:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.868558:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868560:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777088.868561:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777088.868564:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777088.868568:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777088.868569:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777088.868573:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777088.868580:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832576:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775777088.868584:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777088.868587:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868594:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777088.868595:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.868606:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777088.868608:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777088.868610:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777088.868612:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832704:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775777088.868615:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777088.868617:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777088.868618:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777088.868620:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777088.868623:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775777088.868625:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777088.868628:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777088.868630:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777088.868633:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777088.868636:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099832704, portal 4 00000100:00000001:2.0:1775777088.868637:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777088.868638:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629231741757056 : -114841967794560 : ffff978d454de680) 00000100:00000040:2.0:1775777088.868644:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868651:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777088.868653:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777088.868656:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777088.868658:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099832704, offset 0 00000400:00000200:2.0:1775777088.868662:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777088.868666:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.868669:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777088.868672:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777088.868675:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777088.868677:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777088.868679:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777088.868681:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4736:4736:256:4294967295] 192.168.201.139@tcp LPNI seq info [4736:4736:8:4294967295] 00000400:00000200:2.0:1775777088.868685:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777088.868691:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777088.868695:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.868698:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777088.868702:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777088.868704:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1775777088.868708:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777088.868710:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1775777088.868713:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777088.868715:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777088.868718:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777088.868720:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777088.868723:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777088.868724:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777088.868727:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777088.868729:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868736:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.868738:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868744:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.868745:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777088.868748:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777088.868751:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775777088.868753:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777088.868755:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313476621 00000100:00000001:2.0:1775777088.868757:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313477878 00000100:00000001:2.0:1775777088.868759:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313476621 00000100:00000001:2.0:1775777088.868761:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313477878 00000100:00000001:2.0:1775777088.868763:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777088.868765:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.868767:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777088.868771:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777088/real 1775777088] req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777088.868811:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777088.868813:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777088.868823:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777088.868825:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777088.868827:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777088.868829:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777088.868832:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777088.868840:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868846:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777088.868848:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868850:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.868851:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777088.868855:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868863:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777088.868866:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868873:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777088.868876:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868877:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777088.868879:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868880:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777088.868882:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777088.868885:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777088.868888:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777088.868890:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777088.868894:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777088.868902:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d454de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832704:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775777088.868906:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868908:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.868911:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de300 x1862033099832448/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868918:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777088.868920:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777088.868923:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775777088.868926:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777088.868928:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777088.868930:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775777088.868932:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777088.868934:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759dc00. 02000000:00000001:2.0:1775777088.868938:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777088.868940:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777088.868942:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de300. 00000100:00000001:2.0:1775777088.868945:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777088.868946:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868948:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.868951:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454dfb80 x1862033099832576/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868956:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777088.868957:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777088.868959:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506000. 02000000:00000001:2.0:1775777088.868960:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777088.868962:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777088.868963:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775777088.868964:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777088.868965:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e000. 02000000:00000001:2.0:1775777088.868967:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777088.868968:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777088.868969:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454dfb80. 00000100:00000001:2.0:1775777088.868971:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777088.868972:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868974:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777088.868977:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d454de680 x1862033099832704/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777088.868983:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777088.868984:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777088.868985:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51504800. 02000000:00000001:2.0:1775777088.868987:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777088.868988:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777088.868989:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775777088.868990:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777088.868991:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d4759e800. 02000000:00000001:2.0:1775777088.868993:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777088.868994:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777088.868995:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d454de680. 00000100:00000001:2.0:1775777088.868996:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777088.868998:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.868999:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777088.869006:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775777088.869008:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777088.869011:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775777088.869013:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777089.051913:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777089.051919:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777089.051921:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777089.051925:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777089.051928:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777089.051930:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777089.051932:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777089.051934:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777089.051936:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777089.563937:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777089.563960:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777089.563963:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777089.563969:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777089.563972:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777089.563977:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777089.563980:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777089.563982:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777089.563985:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777089.628951:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777089.628957:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777089.628964:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777089.628965:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777089.628969:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777089.628970:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777089.628973:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777089.628974:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777089.629086:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777089.629089:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777089.629092:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777090.075908:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777090.075914:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777090.075916:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777090.075921:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777090.075925:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777090.075928:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777090.075931:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777090.075933:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777090.075936:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777090.139908:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313477878, at 4313477888 00000100:00000040:1.0:1775777090.139908:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313477878, at 4313477888 00000100:00000010:0.0:1775777090.139923:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b4e0. 00000100:00000010:1.0:1775777090.139923:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b8a0. 00000100:00000001:3.0:1775777090.588006:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777090.588013:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777090.588016:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777090.588020:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777090.588024:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777090.588028:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777090.588031:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777090.588045:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777090.588048:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777090.651903:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777090.651909:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777090.651914:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777090.651915:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777090.651919:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777090.651920:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777090.651923:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777090.651924:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777090.652029:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777090.652032:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777090.652035:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777091.099868:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777091.099874:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777091.099876:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777091.099881:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777091.099885:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777091.099888:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777091.099891:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777091.099894:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777091.099896:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777091.611907:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777091.611915:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777091.611917:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777091.611923:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777091.611926:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777091.611930:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777091.611932:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777091.611935:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777091.611937:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777091.677929:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677935:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777091.677938:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777091.677940:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777091.677944:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677946:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777091.677949:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677951:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677953:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677956:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777091.677957:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777091.677958:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777091.677959:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677960:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777091.677961:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677963:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677964:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677968:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777091.677969:0:314648:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1775777091.677970:0:314648:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1775777091.677972:0:314648:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677973:0:314648:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1775777091.677974:0:314648:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677975:0:314648:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777091.677976:0:314648:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777091.678602:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777091.678606:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777091.678609:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777092.124169:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777092.124175:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777092.124177:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777092.124183:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777092.124187:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777092.124190:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777092.124192:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777092.124195:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777092.124197:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777092.635913:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777092.635921:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777092.635922:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777092.635928:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777092.635932:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777092.635936:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777092.635939:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777092.635941:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777092.635944:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777092.699924:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777092.699930:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777092.699936:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777092.699937:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777092.699941:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777092.699942:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777092.699945:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777092.699946:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777092.700124:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777092.700128:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777092.700131:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777093.148022:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777093.148029:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777093.148031:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777093.148037:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777093.148041:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777093.148045:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777093.148047:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777093.148049:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777093.148053:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777093.659945:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777093.659952:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777093.659954:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777093.659959:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777093.659963:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777093.659967:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777093.659969:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777093.659972:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777093.659974:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777093.724096:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777093.724104:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777093.724111:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777093.724112:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777093.724115:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777093.724116:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777093.724119:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777093.724120:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775777093.724321:0:305076:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775777093.724324:0:305076:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775777093.724328:0:305076:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775777093.980046:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) 9df31b64-af5b-4f09-b903-aebce3ef437b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777093.980058:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775777093.980061:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777093.980065:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777093.980069:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777093.980077:0:313512:0:(import.c:509:import_select_connection()) MGC192.168.201.139@tcp: connect to NID 192.168.201.139@tcp last attempt 18809 00000400:00000200:1.0:1775777093.980088:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777093.980095:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777093.980098:0:313512:0:(import.c:588:import_select_connection()) MGC192.168.201.139@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777093.980101:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980116:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980119:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980121:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777093.980126:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777093.980127:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f0fc 00000020:00000001:1.0:1775777093.980130:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777093.980132:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52588800 refcount=3 00000020:00000001:1.0:1775777093.980134:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000020:00000001:1.0:1775777093.980136:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960557568 : -114841748994048 : ffff978d52588800) 00000100:00000040:1.0:1775777093.980139:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980141:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980143:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980144:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777093.980147:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52588800 : new refcount 2 00000100:00080000:1.0:1775777093.980150:0:313512:0:(import.c:623:import_select_connection()) MGC192.168.201.139@tcp: import ffff978d44891800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980153:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777093.980156:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777093.980159:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777093.980162:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980165:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775777093.980167:0:313512:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777093.980176:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ca80. 00000020:00000040:1.0:1775777093.980180:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d44891800 refcount=4 obd=MGC192.168.201.139@tcp 02000000:00000001:1.0:1775777093.980183:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777093.980185:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980188:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777093.980190:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777093.980194:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777093.980202:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3ac00. 00000100:00000001:1.0:1775777093.980208:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777093.980216:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777093.980226:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777093.980230:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777093.980234:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c52f663] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777093.980260:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777093.980263:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777093.980266:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775777093.980268:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777093.980269:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777093.980272:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777093.980275:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18809 00000400:00000200:1.0:1775777093.980279:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777093.980282:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777093.980284:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777093.980286:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980288:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980289:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980291:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777093.980293:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777093.980294:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f150 00000020:00000001:1.0:1775777093.980295:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777093.980297:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258f800 refcount=5 00000020:00000001:1.0:1775777093.980299:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000020:00000001:1.0:1775777093.980300:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960586240 : -114841748965376 : ffff978d5258f800) 00000100:00000040:1.0:1775777093.980303:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980304:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980306:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980308:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777093.980310:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258f800 : new refcount 4 00000100:00080000:1.0:1775777093.980312:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff978d47505000: import ffff978d431d0000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980314:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777093.980315:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777093.980320:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777093.980322:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980324:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777093.980328:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775777093.980330:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980332:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777093.980335:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6df80. 00000020:00000040:1.0:1775777093.980338:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d0000 refcount=309 obd=lustre-MDT0000-mdc-ffff978d47505000 02000000:00000001:1.0:1775777093.980341:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777093.980342:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980344:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777093.980345:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777093.980347:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777093.980350:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f38000. 00000100:00000001:1.0:1775777093.980353:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777093.980357:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777093.980363:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777093.980365:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777093.980368:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ae1941f2] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777093.980375:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777093.980377:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777093.980380:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775777093.980382:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777093.980383:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777093.980385:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777093.980387:0:313512:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18809 00000400:00000200:1.0:1775777093.980391:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777093.980393:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777093.980395:0:313512:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777093.980398:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980400:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980402:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980403:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777093.980405:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777093.980406:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f157 00000020:00000001:1.0:1775777093.980407:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777093.980409:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258b800 refcount=5 00000020:00000001:1.0:1775777093.980411:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000020:00000001:1.0:1775777093.980413:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960569856 : -114841748981760 : ffff978d5258b800) 00000100:00000040:1.0:1775777093.980416:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980418:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980420:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980422:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777093.980424:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258b800 : new refcount 4 00000100:00080000:1.0:1775777093.980426:0:313512:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff978d47505000: import ffff978d431d4000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980428:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777093.980430:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777093.980433:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777093.980434:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980436:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777093.980438:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681674 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775777093.980440:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980442:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777093.980445:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6d500. 00000020:00000040:1.0:1775777093.980447:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d4000 refcount=4 obd=lustre-MDT0001-mdc-ffff978d47505000 02000000:00000001:1.0:1775777093.980450:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777093.980484:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980485:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777093.980487:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777093.980488:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777093.980494:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f39c00. 00000100:00000001:1.0:1775777093.980497:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777093.980501:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777093.980509:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777093.980510:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777093.980514:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b89413d] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777093.980520:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777093.980523:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777093.980526:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1775777093.980528:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777093.980529:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777093.980531:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777093.980534:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18809 00000400:00000200:1.0:1775777093.980538:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777093.980541:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777093.980543:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777093.980545:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980547:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980549:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980552:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777093.980554:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777093.980555:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f165 00000020:00000001:1.0:1775777093.980557:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777093.980558:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d52589000 refcount=4 00000020:00000001:1.0:1775777093.980561:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000020:00000001:1.0:1775777093.980563:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960559616 : -114841748992000 : ffff978d52589000) 00000100:00000040:1.0:1775777093.980565:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980568:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980569:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980571:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777093.980573:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d52589000 : new refcount 3 00000100:00080000:1.0:1775777093.980575:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff978d47505000: import ffff978d431d5000 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980578:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777093.980579:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777093.980582:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777093.980584:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980586:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777093.980589:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775777093.980591:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980592:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777093.980596:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6e680. 00000020:00000040:1.0:1775777093.980598:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d5000 refcount=4 obd=lustre-OST0000-osc-ffff978d47505000 02000000:00000001:1.0:1775777093.980600:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777093.980601:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980602:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777093.980604:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777093.980605:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777093.980608:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3a000. 00000100:00000001:1.0:1775777093.980611:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777093.980616:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777093.980622:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777093.980623:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777093.980627:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cf97f2e] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777093.980635:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775777093.980637:0:313512:0:(pinger.c:229:ptlrpc_pinger_process_import()) eede2d1a-d2b8-4c87-9d71-55318e8159ca->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775777093.980648:0:313512:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1775777093.980649:0:313512:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775777093.980650:0:313512:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775777093.980653:0:313512:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775777093.980655:0:313512:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: connect to NID 192.168.201.139@tcp last attempt 18809 00000400:00000200:1.0:1775777093.980658:0:313512:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:1.0:1775777093.980661:0:313512:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.139@tcp is discovered: rc = 1 00000100:00080000:1.0:1775777093.980663:0:313512:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775777093.980665:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980667:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980669:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980671:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000001:1.0:1775777093.980673:0:313512:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775777093.980674:0:313512:0:(genops.c:823:class_conn2export()) looking for export cookie 0xac0cfa9f84a0f16c 00000020:00000001:1.0:1775777093.980676:0:313512:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775777093.980677:0:313512:0:(lustre_handles.c:131:class_handle2object()) GET export ffff978d5258c800 refcount=4 00000020:00000001:1.0:1775777093.980679:0:313512:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000020:00000001:1.0:1775777093.980681:0:313512:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446629231960573952 : -114841748977664 : ffff978d5258c800) 00000100:00000040:1.0:1775777093.980683:0:313512:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff978d554f65c0 refcount 14 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980684:0:313512:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775777093.980686:0:313512:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff978d554f65c0 refcount 15 to 192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980688:0:313512:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446629232010290624 : -114841699260992 : ffff978d554f65c0) 00000020:00000040:1.0:1775777093.980691:0:313512:0:(genops.c:898:class_export_put()) PUTting export ffff978d5258c800 : new refcount 3 00000100:00080000:1.0:1775777093.980693:0:313512:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff978d47505000: import ffff978d431d1800 using connection 192.168.201.139@tcp/192.168.201.139@tcp 00000100:00000001:1.0:1775777093.980695:0:313512:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775777093.980697:0:313512:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775777093.980699:0:313512:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775777093.980701:0:313512:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980703:0:313512:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775777093.980704:0:313512:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681674 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1775777093.980706:0:313512:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980708:0:313512:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775777093.980711:0:313512:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff978d65d6ea00. 00000020:00000040:1.0:1775777093.980713:0:313512:0:(genops.c:1229:class_import_get()) import ffff978d431d1800 refcount=4 obd=lustre-OST0001-osc-ffff978d47505000 02000000:00000001:1.0:1775777093.980716:0:313512:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775777093.980717:0:313512:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775777093.980718:0:313512:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775777093.980720:0:313512:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775777093.980722:0:313512:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775777093.980725:0:313512:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff978d50f3b000. 00000100:00000001:1.0:1775777093.980728:0:313512:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775777093.980732:0:313512:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775777093.980743:0:313512:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775777093.980745:0:313512:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777093.980748:0:313512:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f1e7a0ee] to pc [ptlrpcd_rcv+-1] req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775777093.980754:0:313512:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775777093.980759:0:313512:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (18819) 00000100:00000001:2.0:1775777093.981270:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775777093.981274:0:301888:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775777093.981277:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777093.981283:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981292:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777093.981294:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.981297:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777093.981299:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777093.981301:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777093.981304:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099832832:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775777093.981308:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777093.981311:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777093.981313:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.981316:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777093.981322:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775777093.981323:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777093.981329:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777093.981333:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777093.981340:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777093.981347:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099832832, portal 25 00000100:00000001:2.0:1775777093.981349:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777093.981351:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287599232 : -114841421952384 : ffff978d65d6ca80) 00000100:00000040:2.0:1775777093.981355:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981362:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777093.981364:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777093.981366:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777093.981369:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862033099832832, offset 0 00000400:00000200:2.0:1775777093.981375:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777093.981383:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.981389:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777093.981394:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777093.981398:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777093.981402:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777093.981404:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777093.981408:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4737:4737:256:4294967295] 192.168.201.139@tcp LPNI seq info [4737:4737:8:4294967295] 00000400:00000200:2.0:1775777093.981413:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.981422:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777093.981427:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.981433:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777093.981439:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777093.981442:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775777093.981448:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777093.981464:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775777093.981468:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.981470:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777093.981478:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777093.981482:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777093.981487:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777093.981489:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777093.981493:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777093.981496:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981505:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.981508:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981514:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.981516:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777093.981521:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777093.981524:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777093.981527:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 0 is 4313482998 00000100:00000010:2.0:1775777093.981532:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398b7e0. 00000100:00000001:2.0:1775777093.981538:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1775777093.981544:0:301888:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS new deadline estimation for cpu 1 is 4313482998 00000100:00000010:2.0:1775777093.981546:0:301888:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff978d4398bd20. 00000100:00000001:2.0:1775777093.981549:0:301888:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.139@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775777093.981552:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777093.981555:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.981557:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777093.981561:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777093/real 1775777093] req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777093.981570:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777093.981572:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777093.981574:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777093.981576:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777093.981577:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777093.981579:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777093.981582:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777093.981595:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981601:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777093.981603:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.981605:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.981606:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777093.981609:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981616:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777093.981620:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981626:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777093.981629:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.981630:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.981631:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.981634:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777093.981636:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777093.981638:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d44891800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777093.981642:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777093.981644:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777093.981648:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777093.981655:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:9df31b64-af5b-4f09-b903-aebce3ef437b:301888:1862033099832832:192.168.201.139@tcp:250:kworker.0 00000100:00000001:2.0:1775777093.981659:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777093.981662:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981667:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777093.981669:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.981671:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777093.981672:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777093.981674:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777093.981677:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832960:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775777093.981680:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777093.981682:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777093.981683:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.981684:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777093.981687:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775777093.981689:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777093.981692:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777093.981695:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777093.981698:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777093.981701:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099832960, portal 10 00000100:00000001:2.0:1775777093.981702:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777093.981703:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287604608 : -114841421947008 : ffff978d65d6df80) 00000100:00000040:2.0:1775777093.981709:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981714:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777093.981716:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777093.981718:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777093.981720:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099832960, offset 0 00000400:00000200:2.0:1775777093.981724:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777093.981728:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.981731:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777093.981734:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777093.981736:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777093.981738:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777093.981740:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777093.981742:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4738:4738:256:4294967295] 192.168.201.139@tcp LPNI seq info [4738:4738:8:4294967295] 00000400:00000200:2.0:1775777093.981746:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.981751:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777093.981756:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.981758:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777093.981762:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777093.981764:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775777093.981768:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777093.981770:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775777093.981773:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.981775:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777093.981811:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777093.981815:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777093.981818:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777093.981820:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777093.981822:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777093.981825:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981832:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.981834:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981840:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.981842:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777093.981844:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777093.981848:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: using a timeout of 3081 usecs (4 jiffies) 00000400:00000200:2.0:1775777093.981850:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777093.981852:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313481734 00000100:00000001:2.0:1775777093.981855:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313482998 00000100:00000001:2.0:1775777093.981857:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313481734 00000100:00000001:2.0:1775777093.981859:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313482998 00000100:00000001:2.0:1775777093.981861:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777093.981862:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.981864:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777093.981867:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777093/real 1775777093] req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777093.981874:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777093.981875:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777093.981877:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777093.981880:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777093.981881:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777093.981883:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777093.981885:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777093.981892:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981897:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777093.981898:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.981900:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.981901:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777093.981904:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981910:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777093.981913:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981918:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777093.981920:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.981921:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.981922:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.981924:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777093.981925:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777093.981927:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d0000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777093.981930:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777093.981932:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777093.981936:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777093.981943:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099832960:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775777093.981946:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777093.981949:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.981956:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777093.981958:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.981959:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777093.981961:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777093.981962:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777093.981965:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099833088:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775777093.981968:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777093.981970:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777093.981971:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.981973:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777093.981976:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775777093.982056:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777093.982059:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777093.982063:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777093.982066:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777093.982069:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099833088, portal 10 00000100:00000001:2.0:1775777093.982071:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777093.982072:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287601920 : -114841421949696 : ffff978d65d6d500) 00000100:00000040:2.0:1775777093.982077:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982083:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777093.982085:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777093.982086:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777093.982088:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862033099833088, offset 0 00000400:00000200:2.0:1775777093.982092:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777093.982096:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.982099:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777093.982102:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777093.982242:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777093.982245:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777093.982247:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777093.982250:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4739:4739:256:4294967295] 192.168.201.139@tcp LPNI seq info [4739:4739:8:4294967295] 00000400:00000200:2.0:1775777093.982254:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.982264:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777093.982270:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.982274:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777093.982278:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777093.982280:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775777093.982285:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777093.982287:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775777093.982291:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.982292:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777093.982295:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777093.982298:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777093.982301:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777093.982303:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777093.982305:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777093.982308:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982315:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.982318:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982323:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.982324:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777093.982327:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777093.982330:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: using a timeout of 4641 usecs (5 jiffies) 00000400:00000200:2.0:1775777093.982333:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777093.982335:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313481735 00000100:00000001:2.0:1775777093.982338:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313482998 00000100:00000001:2.0:1775777093.982340:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313481735 00000100:00000001:2.0:1775777093.982341:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313482998 00000100:00000001:2.0:1775777093.982343:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777093.982345:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.982347:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777093.982351:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777093/real 1775777093] req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777093.982358:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777093.982359:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777093.982361:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777093.982363:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777093.982364:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777093.982366:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777093.982368:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777093.982375:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982381:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777093.982382:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.982384:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.982385:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777093.982388:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982395:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777093.982398:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982404:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777093.982407:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.982408:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.982410:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.982411:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777093.982412:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777093.982415:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d4000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777093.982418:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777093.982420:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777093.982424:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777093.982431:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099833088:192.168.201.139@tcp:38:kworker.0 00000100:00000001:2.0:1775777093.982434:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777093.982437:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982443:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777093.982445:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.982447:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777093.982448:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777093.982450:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777093.982532:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099833216:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775777093.982536:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777093.982538:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777093.982539:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.982541:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777093.982545:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775777093.982546:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777093.982550:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777093.982552:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777093.982555:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777093.982558:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099833216, portal 4 00000100:00000001:2.0:1775777093.982560:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777093.982561:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287606400 : -114841421945216 : ffff978d65d6e680) 00000100:00000040:2.0:1775777093.982566:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982572:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777093.982574:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777093.982576:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777093.982578:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099833216, offset 0 00000400:00000200:2.0:1775777093.982581:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777093.982585:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.982589:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777093.982591:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777093.982593:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777093.982595:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777093.982598:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777093.982601:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4740:4740:256:4294967295] 192.168.201.139@tcp LPNI seq info [4740:4740:8:4294967295] 00000400:00000200:2.0:1775777093.982605:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.982611:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777093.982615:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.982618:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777093.982622:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777093.982624:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775777093.982628:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777093.982630:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775777093.982634:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.982636:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777093.982638:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777093.982642:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777093.982644:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777093.982646:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777093.982648:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777093.982652:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982663:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.982666:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982672:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.982673:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777093.982676:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777093.982679:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: using a timeout of 13072 usecs (14 jiffies) 00000400:00000200:2.0:1775777093.982682:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777093.982683:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313481744 00000100:00000001:2.0:1775777093.982686:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313482998 00000100:00000001:2.0:1775777093.982688:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313481744 00000100:00000001:2.0:1775777093.982690:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313482998 00000100:00000001:2.0:1775777093.982692:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777093.982694:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.982696:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777093.982698:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777093/real 1775777093] req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777093.982706:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777093.982707:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777093.982708:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777093.982710:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777093.982711:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777093.982713:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777093.982715:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777093.982723:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982729:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777093.982731:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.982733:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.982734:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777093.982737:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982744:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777093.982747:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982753:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777093.982755:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.982756:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.982757:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.982758:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777093.982760:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777093.982762:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d5000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777093.982765:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777093.982767:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777093.982770:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777093.982776:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099833216:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775777093.982809:0:301888:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775777093.982813:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982819:0:301888:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775777093.982821:0:301888:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.982823:0:301888:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775777093.982824:0:301888:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775777093.982826:0:301888:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775777093.982829:0:301888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099833344:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775777093.982832:0:301888:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775777093.982833:0:301888:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775777093.982835:0:301888:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775777093.982837:0:301888:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775777093.982840:0:301888:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775777093.982842:0:301888:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775777093.982845:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000010:2.0:1775777093.982847:0:301888:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff978d6601cc60. 00000400:00000010:2.0:1775777093.982850:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d088. 00000100:00000200:2.0:1775777093.982853:0:301888:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862033099833344, portal 4 00000100:00000001:2.0:1775777093.982854:0:301888:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775777093.982855:0:301888:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446629232287607296 : -114841421944320 : ffff978d65d6ea00) 00000100:00000040:2.0:1775777093.982860:0:301888:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982867:0:301888:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775777093.982869:0:301888:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.139@tcp 00000400:00000010:2.0:1775777093.982870:0:301888:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff978d6720d880. 00000100:00000200:2.0:1775777093.982872:0:301888:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862033099833344, offset 0 00000400:00000200:2.0:1775777093.982876:0:301888:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.139@tcp 00000400:00000200:2.0:1775777093.982879:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.982882:0:301888:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.139@tcp local destination 00000400:00000200:2.0:1775777093.982884:0:301888:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775777093.982886:0:301888:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.39@tcp 00000400:00000200:2.0:1775777093.982888:0:301888:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.39@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775777093.982891:0:301888:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.139@tcp 00000400:00000200:2.0:1775777093.982893:0:301888:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.39@tcp NI seq info: [4741:4741:256:4294967295] 192.168.201.139@tcp LPNI seq info [4741:4741:8:4294967295] 00000400:00000200:2.0:1775777093.982897:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000400:00000200:2.0:1775777093.982903:0:301888:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.39@tcp(192.168.201.39@tcp:) -> 192.168.201.139@tcp(192.168.201.139@tcp:192.168.201.139@tcp) : PUT try# 0 00000800:00000200:2.0:1775777093.982906:0:301888:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.982909:0:301888:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff978c46af7f00. 00000800:00000200:2.0:1775777093.982912:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.0:1775777093.982915:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1775777093.982918:0:301888:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff978d66f1d600] -> 12345-192.168.201.139@tcp (2) 00000800:00000200:2.1:1775777093.982921:0:301888:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.139 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1775777093.982924:0:301888:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.139@tcp 00000800:00000010:2.0:1775777093.982926:0:301888:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff978c46af7f00. 00000400:00000200:2.0:1775777093.982929:0:301888:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1775777093.982932:0:301888:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.39@tcp->192.168.201.139@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1775777093.982934:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d880 00000400:00000010:2.0:1775777093.982936:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d880. 00000100:00000001:2.0:1775777093.982938:0:301888:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775777093.982941:0:301888:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982948:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.982950:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 0 dl 1775777118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.982956:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.982958:0:301888:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775777093.982960:0:301888:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775777093.982962:0:301888:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: using a timeout of 4572 usecs (5 jiffies) 00000400:00000200:2.0:1775777093.982966:0:301888:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.139@tcp to cpt 0 00000100:00000001:2.0:1775777093.982967:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 0 is 4313481736 00000100:00000001:2.0:1775777093.982970:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 0, keep current deadline 4313482998 00000100:00000001:2.0:1775777093.982971:0:301888:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS new deadline estimation for cpu 1 is 4313481736 00000100:00000001:2.0:1775777093.982973:0:301888:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff978d47505000: PM QoS request active for cpu 1, keep current deadline 4313482998 00000100:00000001:2.0:1775777093.982975:0:301888:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775777093.982977:0:301888:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.983375:0:301888:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1775777093.983380:0:301888:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775777093/real 1775777093] req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1775777093.983390:0:301888:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1775777093.983391:0:301888:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1775777093.983393:0:301888:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff978d6601cc60. 00000400:00000200:2.0:1775777093.983396:0:301888:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff978d6720d088 00000400:00000010:2.0:1775777093.983397:0:301888:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff978d6720d088. 00000100:00000001:2.0:1775777093.983400:0:301888:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1775777093.983402:0:301888:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1775777093.983410:0:301888:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.983416:0:301888:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775777093.983417:0:301888:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983419:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.983420:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1775777093.983423:0:301888:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.983429:0:301888:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775777093.983432:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.983439:0:301888:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1775777093.983441:0:301888:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983442:0:301888:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1775777093.983443:0:301888:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983445:0:301888:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1775777093.983446:0:301888:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1775777093.983448:0:301888:0:(import.c:65:import_set_state_nolock()) ffff978d431d1800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775777093.983463:0:301888:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.139@tcp failed (-110) 00000100:00000001:2.0:1775777093.983466:0:301888:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1775777093.983469:0:301888:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775777093.983483:0:301888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff978d65d6ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:eede2d1a-d2b8-4c87-9d71-55318e8159ca:301888:1862033099833344:192.168.201.139@tcp:8:kworker.0 00000100:00000001:2.0:1775777093.983486:0:301888:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983489:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.983492:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ca80 x1862033099832832/t0(0) o250->MGC192.168.201.139@tcp@192.168.201.139@tcp:26/25 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.983499:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777093.983501:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777093.983504:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507000. 02000000:00000001:2.0:1775777093.983506:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777093.983509:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777093.983510:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d44891800 refcount=3 obd=MGC192.168.201.139@tcp 00000020:00000001:2.0:1775777093.983512:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777093.983514:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3ac00. 02000000:00000001:2.0:1775777093.983518:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777093.983520:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777093.983521:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ca80. 00000100:00000001:2.0:1775777093.983525:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777093.983526:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983527:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.983531:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6df80 x1862033099832960/t0(0) o38->lustre-MDT0000-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.983537:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777093.983538:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777093.983539:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506800. 02000000:00000001:2.0:1775777093.983541:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777093.983543:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777093.983544:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d0000 refcount=308 obd=lustre-MDT0000-mdc-ffff978d47505000 00000020:00000001:2.0:1775777093.983545:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777093.983547:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f38000. 02000000:00000001:2.0:1775777093.983549:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777093.983550:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777093.983551:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6df80. 00000100:00000001:2.0:1775777093.983554:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777093.983555:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983556:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.983559:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6d500 x1862033099833088/t0(0) o38->lustre-MDT0001-mdc-ffff978d47505000@192.168.201.139@tcp:12/10 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.983565:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777093.983567:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777093.983568:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51507c00. 02000000:00000001:2.0:1775777093.983569:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777093.983570:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777093.983571:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d4000 refcount=3 obd=lustre-MDT0001-mdc-ffff978d47505000 00000020:00000001:2.0:1775777093.983573:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777093.983574:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f39c00. 02000000:00000001:2.0:1775777093.983576:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777093.983576:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777093.983577:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6d500. 00000100:00000001:2.0:1775777093.983579:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777093.983580:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983581:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.983584:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6e680 x1862033099833216/t0(0) o8->lustre-OST0000-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.983589:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777093.983590:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777093.983592:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51506c00. 02000000:00000001:2.0:1775777093.983593:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777093.983594:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777093.983595:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d5000 refcount=3 obd=lustre-OST0000-osc-ffff978d47505000 00000020:00000001:2.0:1775777093.983597:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777093.983598:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3a000. 02000000:00000001:2.0:1775777093.983599:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777093.983600:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777093.983601:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6e680. 00000100:00000001:2.0:1775777093.983603:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777093.983604:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983605:0:301888:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775777093.983608:0:301888:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff978d65d6ea00 x1862033099833344/t0(0) o8->lustre-OST0001-osc-ffff978d47505000@192.168.201.139@tcp:28/4 lens 520/544 e 0 to 1 dl 1775777118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775777093.983614:0:301888:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775777093.983615:0:301888:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775777093.983616:0:301888:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff978d51505c00. 02000000:00000001:2.0:1775777093.983618:0:301888:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775777093.983619:0:301888:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775777093.983620:0:301888:0:(genops.c:1242:class_import_put()) import ffff978d431d1800 refcount=3 obd=lustre-OST0001-osc-ffff978d47505000 00000020:00000001:2.0:1775777093.983621:0:301888:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775777093.983623:0:301888:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff978d50f3b000. 02000000:00000001:2.0:1775777093.983624:0:301888:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775777093.983625:0:301888:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775777093.983626:0:301888:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff978d65d6ea00. 00000100:00000001:2.0:1775777093.983628:0:301888:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775777093.983629:0:301888:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983631:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775777093.983645:0:301888:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775777093.983647:0:301888:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775777093.983650:0:301888:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775777093.983651:0:301888:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777094.171932:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777094.171939:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777094.171941:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777094.171948:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777094.171952:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777094.171956:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777094.171958:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777094.171960:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777094.171963:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775777094.686545:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777094.686552:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777094.686555:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777094.686560:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777094.686564:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777094.686568:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777094.686570:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777094.686573:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777094.686576:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777094.748016:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777094.748022:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777094.748028:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777094.748030:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777094.748033:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777094.748034:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777094.748037:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777094.748038:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777094.748150:0:305077:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777094.748153:0:305077:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777094.748156:0:305077:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777095.199386:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777095.199393:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777095.199395:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777095.199401:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777095.199405:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777095.199409:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777095.199412:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777095.199414:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777095.199417:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775777095.259910:0:314703:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054800 (cpu 0) has reached its deadline 4313482998, at 4313483008 00000100:00000010:0.0:1775777095.259998:0:314703:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398b7e0. 00000100:00000040:1.0:1775777095.260498:0:314632:0:(connection.c:75:cpu_latency_work()) work item of ffff978d43054910 (cpu 0) has reached its deadline 4313482998, at 4313483008 00000100:00000010:1.0:1775777095.260505:0:314632:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff978d4398bd20. 00000100:00000001:3.0:1775777095.707927:0:314194:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777095.707936:0:314194:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777095.707938:0:314194:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777095.707943:0:314194:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d525403c0 going to sleep for 0 seconds 00000100:00000001:3.0:1775777095.707947:0:314194:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777095.707963:0:314194:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777095.707966:0:314194:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777095.707969:0:314194:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777095.707972:0:314194:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777095.773747:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777095.773753:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777095.773758:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777095.773759:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777095.773762:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777095.773763:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775777095.773766:0:314648:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1775777095.773767:0:314648:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775777095.773877:0:312582:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775777095.773881:0:312582:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775777095.773885:0:312582:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775777096.221862:0:313510:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775777096.221871:0:313510:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775777096.221874:0:313510:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775777096.221882:0:313510:0:(client.c:2691:ptlrpc_set_wait()) set ffff978d448d2900 going to sleep for 0 seconds 00000100:00000001:3.0:1775777096.221886:0:313510:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775777096.221891:0:313510:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777096.221894:0:313510:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775777096.221897:0:313510:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775777096.221901:0:313510:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) Debug log: 128723 lines, 128723 kept, 0 dropped, 0 bad.